]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/dma/img-mdc-dma.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86/msr-index: Cleanup bit defines
[mirror_ubuntu-bionic-kernel.git] / drivers / dma / img-mdc-dma.c
CommitLineData
5689ba7f
AB		 1/*
2 * IMG Multi-threaded DMA Controller (MDC)
3 *
4 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
5 * Copyright (C) 2014 Google, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 */
11
12#include <linux/clk.h>
13#include <linux/dma-mapping.h>
14#include <linux/dmaengine.h>
15#include <linux/dmapool.h>
16#include <linux/interrupt.h>
17#include <linux/io.h>
18#include <linux/irq.h>
19#include <linux/kernel.h>
20#include <linux/mfd/syscon.h>
21#include <linux/module.h>
22#include <linux/of.h>
23#include <linux/of_device.h>
24#include <linux/of_dma.h>
25#include <linux/platform_device.h>
56d355e6 26#include <linux/pm_runtime.h>
5689ba7f
AB		 27#include <linux/regmap.h>
28#include <linux/slab.h>
29#include <linux/spinlock.h>
30
31#include "dmaengine.h"
32#include "virt-dma.h"
33
34#define MDC_MAX_DMA_CHANNELS 32
35
36#define MDC_GENERAL_CONFIG 0x000
37#define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)
38#define MDC_GENERAL_CONFIG_IEN BIT(29)
39#define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)
40#define MDC_GENERAL_CONFIG_INC_W BIT(12)
41#define MDC_GENERAL_CONFIG_INC_R BIT(8)
42#define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)
43#define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4
44#define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7
45#define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)
46#define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0
47#define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7
48
49#define MDC_READ_PORT_CONFIG 0x004
50#define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28
51#define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf
52#define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24
53#define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf
54#define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16
55#define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf
56#define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4
57#define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff
58#define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)
59
60#define MDC_READ_ADDRESS 0x008
61
62#define MDC_WRITE_ADDRESS 0x00c
63
64#define MDC_TRANSFER_SIZE 0x010
65#define MDC_TRANSFER_SIZE_MASK 0xffffff
66
67#define MDC_LIST_NODE_ADDRESS 0x014
68
69#define MDC_CMDS_PROCESSED 0x018
70#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
71#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f
72#define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)
73#define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0
74#define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f
75
76#define MDC_CONTROL_AND_STATUS 0x01c
77#define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)
78#define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)
79#define MDC_CONTROL_AND_STATUS_EN BIT(0)
80
81#define MDC_ACTIVE_TRANSFER_SIZE 0x030
82
83#define MDC_GLOBAL_CONFIG_A 0x900
84#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16
85#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff
86#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8
87#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff
88#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0
89#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff
90
91struct mdc_hw_list_desc {
92 u32 gen_conf;
93 u32 readport_conf;
94 u32 read_addr;
95 u32 write_addr;
96 u32 xfer_size;
97 u32 node_addr;
98 u32 cmds_done;
99 u32 ctrl_status;
100 /*
101 * Not part of the list descriptor, but instead used by the CPU to
102 * traverse the list.
103 */
104 struct mdc_hw_list_desc *next_desc;
105};
106
107struct mdc_tx_desc {
108 struct mdc_chan *chan;
109 struct virt_dma_desc vd;
110 dma_addr_t list_phys;
111 struct mdc_hw_list_desc *list;
112 bool cyclic;
113 bool cmd_loaded;
114 unsigned int list_len;
115 unsigned int list_period_len;
116 size_t list_xfer_size;
117 unsigned int list_cmds_done;
118};
119
120struct mdc_chan {
121 struct mdc_dma *mdma;
122 struct virt_dma_chan vc;
123 struct dma_slave_config config;
124 struct mdc_tx_desc *desc;
125 int irq;
126 unsigned int periph;
127 unsigned int thread;
128 unsigned int chan_nr;
129};
130
131struct mdc_dma_soc_data {
132 void (*enable_chan)(struct mdc_chan *mchan);
133 void (*disable_chan)(struct mdc_chan *mchan);
134};
135
136struct mdc_dma {
137 struct dma_device dma_dev;
138 void __iomem *regs;
139 struct clk *clk;
140 struct dma_pool *desc_pool;
141 struct regmap *periph_regs;
142 spinlock_t lock;
143 unsigned int nr_threads;
144 unsigned int nr_channels;
145 unsigned int bus_width;
146 unsigned int max_burst_mult;
147 unsigned int max_xfer_size;
148 const struct mdc_dma_soc_data *soc;
149 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
150};
151
152static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
153{
154 return readl(mdma->regs + reg);
155}
156
157static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
158{
159 writel(val, mdma->regs + reg);
160}
161
162static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
163{
164 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
165}
166
167static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
168{
169 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
170}
171
172static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
173{
174 return container_of(to_virt_chan(c), struct mdc_chan, vc);
175}
176
177static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
178{
179 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
180
181 return container_of(vdesc, struct mdc_tx_desc, vd);
182}
183
184static inline struct device *mdma2dev(struct mdc_dma *mdma)
185{
186 return mdma->dma_dev.dev;
187}
188
189static inline unsigned int to_mdc_width(unsigned int bytes)
190{
191 return ffs(bytes) - 1;
192}
193
194static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
195 unsigned int bytes)
196{
197 ldesc->gen_conf |= to_mdc_width(bytes) <<
198 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
199}
200
201static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
202 unsigned int bytes)
203{
204 ldesc->gen_conf |= to_mdc_width(bytes) <<
205 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
206}
207
208static void mdc_list_desc_config(struct mdc_chan *mchan,
209 struct mdc_hw_list_desc *ldesc,
210 enum dma_transfer_direction dir,
211 dma_addr_t src, dma_addr_t dst, size_t len)
212{
213 struct mdc_dma *mdma = mchan->mdma;
214 unsigned int max_burst, burst_size;
215
216 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
217 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
218 MDC_GENERAL_CONFIG_PHYSICAL_R;
219 ldesc->readport_conf =
220 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
221 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
222 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
223 ldesc->read_addr = src;
224 ldesc->write_addr = dst;
225 ldesc->xfer_size = len - 1;
226 ldesc->node_addr = 0;
227 ldesc->cmds_done = 0;
228 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
229 MDC_CONTROL_AND_STATUS_EN;
230 ldesc->next_desc = NULL;
231
232 if (IS_ALIGNED(dst, mdma->bus_width) &&
233 IS_ALIGNED(src, mdma->bus_width))
234 max_burst = mdma->bus_width * mdma->max_burst_mult;
235 else
236 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
237
238 if (dir == DMA_MEM_TO_DEV) {
239 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
240 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
241 mdc_set_read_width(ldesc, mdma->bus_width);
242 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
243 burst_size = min(max_burst, mchan->config.dst_maxburst *
244 mchan->config.dst_addr_width);
245 } else if (dir == DMA_DEV_TO_MEM) {
246 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
247 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
248 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
249 mdc_set_write_width(ldesc, mdma->bus_width);
250 burst_size = min(max_burst, mchan->config.src_maxburst *
251 mchan->config.src_addr_width);
252 } else {
253 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
254 MDC_GENERAL_CONFIG_INC_W;
255 mdc_set_read_width(ldesc, mdma->bus_width);
256 mdc_set_write_width(ldesc, mdma->bus_width);
257 burst_size = max_burst;
258 }
259 ldesc->readport_conf |= (burst_size - 1) <<
260 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
261}
262
263static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
264{
265 struct mdc_dma *mdma = mdesc->chan->mdma;
266 struct mdc_hw_list_desc *curr, *next;
267 dma_addr_t curr_phys, next_phys;
268
269 curr = mdesc->list;
270 curr_phys = mdesc->list_phys;
271 while (curr) {
272 next = curr->next_desc;
273 next_phys = curr->node_addr;
274 dma_pool_free(mdma->desc_pool, curr, curr_phys);
275 curr = next;
276 curr_phys = next_phys;
277 }
278}
279
280static void mdc_desc_free(struct virt_dma_desc *vd)
281{
282 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
283
284 mdc_list_desc_free(mdesc);
285 kfree(mdesc);
286}
287
288static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
289 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
290 unsigned long flags)
291{
292 struct mdc_chan *mchan = to_mdc_chan(chan);
293 struct mdc_dma *mdma = mchan->mdma;
294 struct mdc_tx_desc *mdesc;
295 struct mdc_hw_list_desc *curr, *prev = NULL;
e5a6b3d5 296 dma_addr_t curr_phys;
5689ba7f
AB		 297
298 if (!len)
299 return NULL;
300
301 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
302 if (!mdesc)
303 return NULL;
304 mdesc->chan = mchan;
305 mdesc->list_xfer_size = len;
306
307 while (len > 0) {
308 size_t xfer_size;
309
310 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
311 if (!curr)
312 goto free_desc;
313
314 if (prev) {
315 prev->node_addr = curr_phys;
316 prev->next_desc = curr;
317 } else {
318 mdesc->list_phys = curr_phys;
319 mdesc->list = curr;
320 }
321
322 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
323
324 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
325 xfer_size);
326
327 prev = curr;
5689ba7f
AB		 328
329 mdesc->list_len++;
330 src += xfer_size;
331 dest += xfer_size;
332 len -= xfer_size;
333 }
334
335 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
336
337free_desc:
338 mdc_desc_free(&mdesc->vd);
339
340 return NULL;
341}
342
343static int mdc_check_slave_width(struct mdc_chan *mchan,
344 enum dma_transfer_direction dir)
345{
346 enum dma_slave_buswidth width;
347
348 if (dir == DMA_MEM_TO_DEV)
349 width = mchan->config.dst_addr_width;
350 else
351 width = mchan->config.src_addr_width;
352
353 switch (width) {
354 case DMA_SLAVE_BUSWIDTH_1_BYTE:
355 case DMA_SLAVE_BUSWIDTH_2_BYTES:
356 case DMA_SLAVE_BUSWIDTH_4_BYTES:
357 case DMA_SLAVE_BUSWIDTH_8_BYTES:
358 break;
359 default:
360 return -EINVAL;
361 }
362
363 if (width > mchan->mdma->bus_width)
364 return -EINVAL;
365
366 return 0;
367}
368
369static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
370 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
371 size_t period_len, enum dma_transfer_direction dir,
372 unsigned long flags)
373{
374 struct mdc_chan *mchan = to_mdc_chan(chan);
375 struct mdc_dma *mdma = mchan->mdma;
376 struct mdc_tx_desc *mdesc;
377 struct mdc_hw_list_desc *curr, *prev = NULL;
e5a6b3d5 378 dma_addr_t curr_phys;
5689ba7f
AB		 379
380 if (!buf_len && !period_len)
381 return NULL;
382
383 if (!is_slave_direction(dir))
384 return NULL;
385
386 if (mdc_check_slave_width(mchan, dir) < 0)
387 return NULL;
388
389 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
390 if (!mdesc)
391 return NULL;
392 mdesc->chan = mchan;
393 mdesc->cyclic = true;
394 mdesc->list_xfer_size = buf_len;
395 mdesc->list_period_len = DIV_ROUND_UP(period_len,
396 mdma->max_xfer_size);
397
398 while (buf_len > 0) {
399 size_t remainder = min(period_len, buf_len);
400
401 while (remainder > 0) {
402 size_t xfer_size;
403
404 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
405 &curr_phys);
406 if (!curr)
407 goto free_desc;
408
409 if (!prev) {
410 mdesc->list_phys = curr_phys;
411 mdesc->list = curr;
412 } else {
413 prev->node_addr = curr_phys;
414 prev->next_desc = curr;
415 }
416
417 xfer_size = min_t(size_t, mdma->max_xfer_size,
418 remainder);
419
420 if (dir == DMA_MEM_TO_DEV) {
421 mdc_list_desc_config(mchan, curr, dir,
422 buf_addr,
423 mchan->config.dst_addr,
424 xfer_size);
425 } else {
426 mdc_list_desc_config(mchan, curr, dir,
427 mchan->config.src_addr,
428 buf_addr,
429 xfer_size);
430 }
431
432 prev = curr;
5689ba7f
AB		 433
434 mdesc->list_len++;
435 buf_addr += xfer_size;
436 buf_len -= xfer_size;
437 remainder -= xfer_size;
438 }
439 }
440 prev->node_addr = mdesc->list_phys;
441
442 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
443
444free_desc:
445 mdc_desc_free(&mdesc->vd);
446
447 return NULL;
448}
449
450static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
451 struct dma_chan *chan, struct scatterlist *sgl,
452 unsigned int sg_len, enum dma_transfer_direction dir,
453 unsigned long flags, void *context)
454{
455 struct mdc_chan *mchan = to_mdc_chan(chan);
456 struct mdc_dma *mdma = mchan->mdma;
457 struct mdc_tx_desc *mdesc;
458 struct scatterlist *sg;
459 struct mdc_hw_list_desc *curr, *prev = NULL;
e5a6b3d5 460 dma_addr_t curr_phys;
5689ba7f
AB		 461 unsigned int i;
462
463 if (!sgl)
464 return NULL;
465
466 if (!is_slave_direction(dir))
467 return NULL;
468
469 if (mdc_check_slave_width(mchan, dir) < 0)
470 return NULL;
471
472 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
473 if (!mdesc)
474 return NULL;
475 mdesc->chan = mchan;
476
477 for_each_sg(sgl, sg, sg_len, i) {
478 dma_addr_t buf = sg_dma_address(sg);
479 size_t buf_len = sg_dma_len(sg);
480
481 while (buf_len > 0) {
482 size_t xfer_size;
483
484 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
485 &curr_phys);
486 if (!curr)
487 goto free_desc;
488
489 if (!prev) {
490 mdesc->list_phys = curr_phys;
491 mdesc->list = curr;
492 } else {
493 prev->node_addr = curr_phys;
494 prev->next_desc = curr;
495 }
496
497 xfer_size = min_t(size_t, mdma->max_xfer_size,
498 buf_len);
499
500 if (dir == DMA_MEM_TO_DEV) {
501 mdc_list_desc_config(mchan, curr, dir, buf,
502 mchan->config.dst_addr,
503 xfer_size);
504 } else {
505 mdc_list_desc_config(mchan, curr, dir,
506 mchan->config.src_addr,
507 buf, xfer_size);
508 }
509
510 prev = curr;
5689ba7f
AB		 511
512 mdesc->list_len++;
513 mdesc->list_xfer_size += xfer_size;
514 buf += xfer_size;
515 buf_len -= xfer_size;
516 }
517 }
518
519 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
520
521free_desc:
522 mdc_desc_free(&mdesc->vd);
523
524 return NULL;
525}
526
527static void mdc_issue_desc(struct mdc_chan *mchan)
528{
529 struct mdc_dma *mdma = mchan->mdma;
530 struct virt_dma_desc *vd;
531 struct mdc_tx_desc *mdesc;
532 u32 val;
533
534 vd = vchan_next_desc(&mchan->vc);
535 if (!vd)
536 return;
537
538 list_del(&vd->node);
539
540 mdesc = to_mdc_desc(&vd->tx);
541 mchan->desc = mdesc;
542
543 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
544 mchan->chan_nr);
545
546 mdma->soc->enable_chan(mchan);
547
548 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
549 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
550 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
551 MDC_GENERAL_CONFIG_PHYSICAL_R;
552 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
553 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
554 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
555 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
556 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
557 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
558 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
559 val |= MDC_CONTROL_AND_STATUS_LIST_EN;
560 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
561}
562
563static void mdc_issue_pending(struct dma_chan *chan)
564{
565 struct mdc_chan *mchan = to_mdc_chan(chan);
566 unsigned long flags;
567
568 spin_lock_irqsave(&mchan->vc.lock, flags);
569 if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
570 mdc_issue_desc(mchan);
571 spin_unlock_irqrestore(&mchan->vc.lock, flags);
572}
573
574static enum dma_status mdc_tx_status(struct dma_chan *chan,
575 dma_cookie_t cookie, struct dma_tx_state *txstate)
576{
577 struct mdc_chan *mchan = to_mdc_chan(chan);
578 struct mdc_tx_desc *mdesc;
579 struct virt_dma_desc *vd;
580 unsigned long flags;
581 size_t bytes = 0;
582 int ret;
583
584 ret = dma_cookie_status(chan, cookie, txstate);
585 if (ret == DMA_COMPLETE)
586 return ret;
587
588 if (!txstate)
589 return ret;
590
591 spin_lock_irqsave(&mchan->vc.lock, flags);
592 vd = vchan_find_desc(&mchan->vc, cookie);
593 if (vd) {
594 mdesc = to_mdc_desc(&vd->tx);
595 bytes = mdesc->list_xfer_size;
596 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
597 struct mdc_hw_list_desc *ldesc;
598 u32 val1, val2, done, processed, residue;
599 int i, cmds;
600
601 mdesc = mchan->desc;
602
603 /*
604 * Determine the number of commands that haven't been
605 * processed (handled by the IRQ handler) yet.
606 */
607 do {
608 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
609 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
610 residue = mdc_chan_readl(mchan,
611 MDC_ACTIVE_TRANSFER_SIZE);
612 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
613 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
614 } while (val1 != val2);
615
616 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
617 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
618 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
619 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
620 cmds = (done - processed) %
621 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
622
623 /*
624 * If the command loaded event hasn't been processed yet, then
625 * the difference above includes an extra command.
626 */
627 if (!mdesc->cmd_loaded)
628 cmds--;
629 else
630 cmds += mdesc->list_cmds_done;
631
632 bytes = mdesc->list_xfer_size;
633 ldesc = mdesc->list;
634 for (i = 0; i < cmds; i++) {
635 bytes -= ldesc->xfer_size + 1;
636 ldesc = ldesc->next_desc;
637 }
638 if (ldesc) {
639 if (residue != MDC_TRANSFER_SIZE_MASK)
640 bytes -= ldesc->xfer_size - residue;
641 else
642 bytes -= ldesc->xfer_size + 1;
643 }
644 }
645 spin_unlock_irqrestore(&mchan->vc.lock, flags);
646
647 dma_set_residue(txstate, bytes);
648
649 return ret;
650}
651
0c328de7
DH		 652static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
653{
654 u32 val, processed, done1, done2;
655 unsigned int ret;
656
657 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
658 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
659 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
660 /*
661 * CMDS_DONE may have incremented between reading CMDS_PROCESSED
662 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we
663 * didn't miss a command completion.
664 */
665 do {
666 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
667
668 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
669 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
670
671 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
672 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
673 MDC_CMDS_PROCESSED_INT_ACTIVE);
674
675 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
676
677 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
678
679 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
680
681 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
682 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
683 } while (done1 != done2);
684
685 if (done1 >= processed)
686 ret = done1 - processed;
687 else
688 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
689 processed) + done1;
690
691 return ret;
692}
693
5689ba7f
AB		 694static int mdc_terminate_all(struct dma_chan *chan)
695{
696 struct mdc_chan *mchan = to_mdc_chan(chan);
697 struct mdc_tx_desc *mdesc;
698 unsigned long flags;
699 LIST_HEAD(head);
700
701 spin_lock_irqsave(&mchan->vc.lock, flags);
702
703 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
704 MDC_CONTROL_AND_STATUS);
705
706 mdesc = mchan->desc;
707 mchan->desc = NULL;
708 vchan_get_all_descriptors(&mchan->vc, &head);
709
0c328de7
DH		 710 mdc_get_new_events(mchan);
711
5689ba7f
AB		 712 spin_unlock_irqrestore(&mchan->vc.lock, flags);
713
714 if (mdesc)
715 mdc_desc_free(&mdesc->vd);
716 vchan_dma_desc_free_list(&mchan->vc, &head);
717
718 return 0;
719}
720
721static int mdc_slave_config(struct dma_chan *chan,
722 struct dma_slave_config *config)
723{
724 struct mdc_chan *mchan = to_mdc_chan(chan);
725 unsigned long flags;
726
727 spin_lock_irqsave(&mchan->vc.lock, flags);
728 mchan->config = *config;
729 spin_unlock_irqrestore(&mchan->vc.lock, flags);
730
731 return 0;
732}
733
56d355e6
EB		 734static int mdc_alloc_chan_resources(struct dma_chan *chan)
735{
736 struct mdc_chan *mchan = to_mdc_chan(chan);
737 struct device *dev = mdma2dev(mchan->mdma);
738
739 return pm_runtime_get_sync(dev);
740}
741
5689ba7f
AB		 742static void mdc_free_chan_resources(struct dma_chan *chan)
743{
744 struct mdc_chan *mchan = to_mdc_chan(chan);
745 struct mdc_dma *mdma = mchan->mdma;
56d355e6 746 struct device *dev = mdma2dev(mdma);
5689ba7f
AB		 747
748 mdc_terminate_all(chan);
5689ba7f 749 mdma->soc->disable_chan(mchan);
56d355e6 750 pm_runtime_put(dev);
5689ba7f
AB		 751}
752
753static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
754{
755 struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
756 struct mdc_tx_desc *mdesc;
0c328de7 757 unsigned int i, new_events;
5689ba7f
AB		 758
759 spin_lock(&mchan->vc.lock);
760
5689ba7f
AB		 761 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
762
0c328de7
DH		 763 new_events = mdc_get_new_events(mchan);
764
765 if (!new_events)
766 goto out;
767
5689ba7f
AB		 768 mdesc = mchan->desc;
769 if (!mdesc) {
770 dev_warn(mdma2dev(mchan->mdma),
771 "IRQ with no active descriptor on channel %d\n",
772 mchan->chan_nr);
773 goto out;
774 }
775
0c328de7 776 for (i = 0; i < new_events; i++) {
5689ba7f
AB		 777 /*
778 * The first interrupt in a transfer indicates that the
779 * command list has been loaded, not that a command has
780 * been completed.
781 */
782 if (!mdesc->cmd_loaded) {
783 mdesc->cmd_loaded = true;
784 continue;
785 }
786
787 mdesc->list_cmds_done++;
788 if (mdesc->cyclic) {
789 mdesc->list_cmds_done %= mdesc->list_len;
790 if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
791 vchan_cyclic_callback(&mdesc->vd);
792 } else if (mdesc->list_cmds_done == mdesc->list_len) {
793 mchan->desc = NULL;
794 vchan_cookie_complete(&mdesc->vd);
795 mdc_issue_desc(mchan);
796 break;
797 }
798 }
799out:
800 spin_unlock(&mchan->vc.lock);
801
802 return IRQ_HANDLED;
803}
804
805static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
806 struct of_dma *ofdma)
807{
808 struct mdc_dma *mdma = ofdma->of_dma_data;
809 struct dma_chan *chan;
810
811 if (dma_spec->args_count != 3)
812 return NULL;
813
814 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
815 struct mdc_chan *mchan = to_mdc_chan(chan);
816
817 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
818 continue;
819 if (dma_get_slave_channel(chan)) {
820 mchan->periph = dma_spec->args[0];
821 mchan->thread = dma_spec->args[2];
822 return chan;
823 }
824 }
825
826 return NULL;
827}
828
829#define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))
830#define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
831#define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f
832
833static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
834{
835 struct mdc_dma *mdma = mchan->mdma;
836
837 regmap_update_bits(mdma->periph_regs,
838 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
839 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
840 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
841 mchan->periph <<
842 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
843}
844
845static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
846{
847 struct mdc_dma *mdma = mchan->mdma;
848
849 regmap_update_bits(mdma->periph_regs,
850 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
851 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
852 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
853 0);
854}
855
856static const struct mdc_dma_soc_data pistachio_mdc_data = {
857 .enable_chan = pistachio_mdc_enable_chan,
858 .disable_chan = pistachio_mdc_disable_chan,
859};
860
861static const struct of_device_id mdc_dma_of_match[] = {
862 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
863 { },
864};
865MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
866
56d355e6
EB		 867static int img_mdc_runtime_suspend(struct device *dev)
868{
869 struct mdc_dma *mdma = dev_get_drvdata(dev);
870
871 clk_disable_unprepare(mdma->clk);
872
873 return 0;
874}
875
876static int img_mdc_runtime_resume(struct device *dev)
877{
878 struct mdc_dma *mdma = dev_get_drvdata(dev);
879
880 return clk_prepare_enable(mdma->clk);
881}
882
5689ba7f
AB		 883static int mdc_dma_probe(struct platform_device *pdev)
884{
885 struct mdc_dma *mdma;
886 struct resource *res;
5689ba7f
AB		 887 unsigned int i;
888 u32 val;
889 int ret;
890
891 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
892 if (!mdma)
893 return -ENOMEM;
894 platform_set_drvdata(pdev, mdma);
895
32e80820 896 mdma->soc = of_device_get_match_data(&pdev->dev);
5689ba7f
AB		 897
898 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
899 mdma->regs = devm_ioremap_resource(&pdev->dev, res);
900 if (IS_ERR(mdma->regs))
901 return PTR_ERR(mdma->regs);
902
903 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
904 "img,cr-periph");
905 if (IS_ERR(mdma->periph_regs))
906 return PTR_ERR(mdma->periph_regs);
907
908 mdma->clk = devm_clk_get(&pdev->dev, "sys");
909 if (IS_ERR(mdma->clk))
910 return PTR_ERR(mdma->clk);
911
5689ba7f
AB		 912 dma_cap_zero(mdma->dma_dev.cap_mask);
913 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
914 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
915 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
916 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
917
918 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
919 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
920 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
921 mdma->nr_threads =
922 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
923 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
924 mdma->bus_width =
925 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
926 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
927 /*
928 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
929 * are supported, this makes it possible for the value reported in
930 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
931 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
932 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this
933 * ambiguity, restrict transfer sizes to one bus-width less than the
934 * actual maximum.
935 */
936 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
937
938 of_property_read_u32(pdev->dev.of_node, "dma-channels",
939 &mdma->nr_channels);
940 ret = of_property_read_u32(pdev->dev.of_node,
941 "img,max-burst-multiplier",
942 &mdma->max_burst_mult);
943 if (ret)
56d355e6 944 return ret;
5689ba7f
AB		 945
946 mdma->dma_dev.dev = &pdev->dev;
947 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
948 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
949 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
56d355e6 950 mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
5689ba7f
AB		 951 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
952 mdma->dma_dev.device_tx_status = mdc_tx_status;
953 mdma->dma_dev.device_issue_pending = mdc_issue_pending;
954 mdma->dma_dev.device_terminate_all = mdc_terminate_all;
955 mdma->dma_dev.device_config = mdc_slave_config;
956
957 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
958 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
959 for (i = 1; i <= mdma->bus_width; i <<= 1) {
960 mdma->dma_dev.src_addr_widths |= BIT(i);
961 mdma->dma_dev.dst_addr_widths |= BIT(i);
962 }
963
964 INIT_LIST_HEAD(&mdma->dma_dev.channels);
965 for (i = 0; i < mdma->nr_channels; i++) {
966 struct mdc_chan *mchan = &mdma->channels[i];
967
968 mchan->mdma = mdma;
969 mchan->chan_nr = i;
970 mchan->irq = platform_get_irq(pdev, i);
56d355e6
EB		 971 if (mchan->irq < 0)
972 return mchan->irq;
973
5689ba7f
AB		 974 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
975 IRQ_TYPE_LEVEL_HIGH,
976 dev_name(&pdev->dev), mchan);
977 if (ret < 0)
56d355e6 978 return ret;
5689ba7f
AB		 979
980 mchan->vc.desc_free = mdc_desc_free;
981 vchan_init(&mchan->vc, &mdma->dma_dev);
982 }
983
984 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
985 sizeof(struct mdc_hw_list_desc),
986 4, 0);
56d355e6
EB		 987 if (!mdma->desc_pool)
988 return -ENOMEM;
989
990 pm_runtime_enable(&pdev->dev);
991 if (!pm_runtime_enabled(&pdev->dev)) {
992 ret = img_mdc_runtime_resume(&pdev->dev);
993 if (ret)
994 return ret;
5689ba7f
AB		 995 }
996
997 ret = dma_async_device_register(&mdma->dma_dev);
998 if (ret)
56d355e6 999 goto suspend;
5689ba7f
AB		 1000
1001 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1002 if (ret)
1003 goto unregister;
1004
1005 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1006 mdma->nr_channels, mdma->nr_threads);
1007
1008 return 0;
1009
1010unregister:
1011 dma_async_device_unregister(&mdma->dma_dev);
56d355e6
EB		 1012suspend:
1013 if (!pm_runtime_enabled(&pdev->dev))
1014 img_mdc_runtime_suspend(&pdev->dev);
1015 pm_runtime_disable(&pdev->dev);
5689ba7f
AB		 1016 return ret;
1017}
1018
1019static int mdc_dma_remove(struct platform_device *pdev)
1020{
1021 struct mdc_dma *mdma = platform_get_drvdata(pdev);
1022 struct mdc_chan *mchan, *next;
1023
1024 of_dma_controller_free(pdev->dev.of_node);
1025 dma_async_device_unregister(&mdma->dma_dev);
1026
1027 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1028 vc.chan.device_node) {
1029 list_del(&mchan->vc.chan.device_node);
1030
5689ba7f
AB		 1031 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1032
1033 tasklet_kill(&mchan->vc.task);
1034 }
1035
56d355e6
EB		 1036 pm_runtime_disable(&pdev->dev);
1037 if (!pm_runtime_status_suspended(&pdev->dev))
1038 img_mdc_runtime_suspend(&pdev->dev);
5689ba7f
AB		 1039
1040 return 0;
1041}
1042
fd9f22ae
EB		 1043#ifdef CONFIG_PM_SLEEP
1044static int img_mdc_suspend_late(struct device *dev)
1045{
1046 struct mdc_dma *mdma = dev_get_drvdata(dev);
1047 int i;
1048
1049 /* Check that all channels are idle */
1050 for (i = 0; i < mdma->nr_channels; i++) {
1051 struct mdc_chan *mchan = &mdma->channels[i];
1052
1053 if (unlikely(mchan->desc))
1054 return -EBUSY;
1055 }
1056
56d355e6 1057 return pm_runtime_force_suspend(dev);
fd9f22ae
EB		 1058}
1059
1060static int img_mdc_resume_early(struct device *dev)
1061{
56d355e6 1062 return pm_runtime_force_resume(dev);
fd9f22ae
EB		 1063}
1064#endif /* CONFIG_PM_SLEEP */
1065
1066static const struct dev_pm_ops img_mdc_pm_ops = {
56d355e6
EB		 1067 SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1068 img_mdc_runtime_resume, NULL)
fd9f22ae
EB		 1069 SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1070 img_mdc_resume_early)
1071};
1072
5689ba7f
AB		 1073static struct platform_driver mdc_dma_driver = {
1074 .driver = {
1075 .name = "img-mdc-dma",
fd9f22ae 1076 .pm = &img_mdc_pm_ops,
5689ba7f
AB		 1077 .of_match_table = of_match_ptr(mdc_dma_of_match),
1078 },
1079 .probe = mdc_dma_probe,
1080 .remove = mdc_dma_remove,
1081};
1082module_platform_driver(mdc_dma_driver);
1083
1084MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1085MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1086MODULE_LICENSE("GPL v2");
ubuntu-bionic-kernel mirror