]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/dma/sprd-dma.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: sprd: Fix the possible memory leak issue
[mirror_ubuntu-bionic-kernel.git] / drivers / dma / sprd-dma.c
CommitLineData
9b3b8171
BW		 1/*
2 * Copyright (C) 2017 Spreadtrum Communications Inc.
3 *
4 * SPDX-License-Identifier: GPL-2.0
5 */
6
7#include <linux/clk.h>
8#include <linux/dma-mapping.h>
9#include <linux/errno.h>
10#include <linux/init.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/kernel.h>
14#include <linux/module.h>
15#include <linux/of.h>
16#include <linux/of_dma.h>
17#include <linux/of_device.h>
18#include <linux/pm_runtime.h>
19#include <linux/slab.h>
20
21#include "virt-dma.h"
22
23#define SPRD_DMA_CHN_REG_OFFSET 0x1000
24#define SPRD_DMA_CHN_REG_LENGTH 0x40
25#define SPRD_DMA_MEMCPY_MIN_SIZE 64
26
27/* DMA global registers definition */
28#define SPRD_DMA_GLB_PAUSE 0x0
29#define SPRD_DMA_GLB_FRAG_WAIT 0x4
30#define SPRD_DMA_GLB_REQ_PEND0_EN 0x8
31#define SPRD_DMA_GLB_REQ_PEND1_EN 0xc
32#define SPRD_DMA_GLB_INT_RAW_STS 0x10
33#define SPRD_DMA_GLB_INT_MSK_STS 0x14
34#define SPRD_DMA_GLB_REQ_STS 0x18
35#define SPRD_DMA_GLB_CHN_EN_STS 0x1c
36#define SPRD_DMA_GLB_DEBUG_STS 0x20
37#define SPRD_DMA_GLB_ARB_SEL_STS 0x24
38#define SPRD_DMA_GLB_REQ_UID(uid) (0x4 * ((uid) - 1))
39#define SPRD_DMA_GLB_REQ_UID_OFFSET 0x2000
40
41/* DMA channel registers definition */
42#define SPRD_DMA_CHN_PAUSE 0x0
43#define SPRD_DMA_CHN_REQ 0x4
44#define SPRD_DMA_CHN_CFG 0x8
45#define SPRD_DMA_CHN_INTC 0xc
46#define SPRD_DMA_CHN_SRC_ADDR 0x10
47#define SPRD_DMA_CHN_DES_ADDR 0x14
48#define SPRD_DMA_CHN_FRG_LEN 0x18
49#define SPRD_DMA_CHN_BLK_LEN 0x1c
50#define SPRD_DMA_CHN_TRSC_LEN 0x20
51#define SPRD_DMA_CHN_TRSF_STEP 0x24
52#define SPRD_DMA_CHN_WARP_PTR 0x28
53#define SPRD_DMA_CHN_WARP_TO 0x2c
54#define SPRD_DMA_CHN_LLIST_PTR 0x30
55#define SPRD_DMA_CHN_FRAG_STEP 0x34
56#define SPRD_DMA_CHN_SRC_BLK_STEP 0x38
57#define SPRD_DMA_CHN_DES_BLK_STEP 0x3c
58
59/* SPRD_DMA_CHN_INTC register definition */
60#define SPRD_DMA_INT_MASK GENMASK(4, 0)
61#define SPRD_DMA_INT_CLR_OFFSET 24
62#define SPRD_DMA_FRAG_INT_EN BIT(0)
63#define SPRD_DMA_BLK_INT_EN BIT(1)
64#define SPRD_DMA_TRANS_INT_EN BIT(2)
65#define SPRD_DMA_LIST_INT_EN BIT(3)
66#define SPRD_DMA_CFG_ERR_INT_EN BIT(4)
67
68/* SPRD_DMA_CHN_CFG register definition */
69#define SPRD_DMA_CHN_EN BIT(0)
70#define SPRD_DMA_WAIT_BDONE_OFFSET 24
71#define SPRD_DMA_DONOT_WAIT_BDONE 1
72
73/* SPRD_DMA_CHN_REQ register definition */
74#define SPRD_DMA_REQ_EN BIT(0)
75
76/* SPRD_DMA_CHN_PAUSE register definition */
77#define SPRD_DMA_PAUSE_EN BIT(0)
78#define SPRD_DMA_PAUSE_STS BIT(2)
79#define SPRD_DMA_PAUSE_CNT 0x2000
80
81/* DMA_CHN_WARP_* register definition */
82#define SPRD_DMA_HIGH_ADDR_MASK GENMASK(31, 28)
83#define SPRD_DMA_LOW_ADDR_MASK GENMASK(31, 0)
84#define SPRD_DMA_HIGH_ADDR_OFFSET 4
85
86/* SPRD_DMA_CHN_INTC register definition */
87#define SPRD_DMA_FRAG_INT_STS BIT(16)
88#define SPRD_DMA_BLK_INT_STS BIT(17)
89#define SPRD_DMA_TRSC_INT_STS BIT(18)
90#define SPRD_DMA_LIST_INT_STS BIT(19)
91#define SPRD_DMA_CFGERR_INT_STS BIT(20)
92#define SPRD_DMA_CHN_INT_STS \
93 (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS | \
94 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS | \
95 SPRD_DMA_CFGERR_INT_STS)
96
97/* SPRD_DMA_CHN_FRG_LEN register definition */
98#define SPRD_DMA_SRC_DATAWIDTH_OFFSET 30
99#define SPRD_DMA_DES_DATAWIDTH_OFFSET 28
100#define SPRD_DMA_SWT_MODE_OFFSET 26
101#define SPRD_DMA_REQ_MODE_OFFSET 24
102#define SPRD_DMA_REQ_MODE_MASK GENMASK(1, 0)
103#define SPRD_DMA_FIX_SEL_OFFSET 21
104#define SPRD_DMA_FIX_EN_OFFSET 20
105#define SPRD_DMA_LLIST_END_OFFSET 19
106#define SPRD_DMA_FRG_LEN_MASK GENMASK(16, 0)
107
108/* SPRD_DMA_CHN_BLK_LEN register definition */
109#define SPRD_DMA_BLK_LEN_MASK GENMASK(16, 0)
110
111/* SPRD_DMA_CHN_TRSC_LEN register definition */
112#define SPRD_DMA_TRSC_LEN_MASK GENMASK(27, 0)
113
114/* SPRD_DMA_CHN_TRSF_STEP register definition */
115#define SPRD_DMA_DEST_TRSF_STEP_OFFSET 16
116#define SPRD_DMA_SRC_TRSF_STEP_OFFSET 0
117#define SPRD_DMA_TRSF_STEP_MASK GENMASK(15, 0)
118
119#define SPRD_DMA_SOFTWARE_UID 0
120
121/*
122 * enum sprd_dma_req_mode: define the DMA request mode
123 * @SPRD_DMA_FRAG_REQ: fragment request mode
124 * @SPRD_DMA_BLK_REQ: block request mode
125 * @SPRD_DMA_TRANS_REQ: transaction request mode
126 * @SPRD_DMA_LIST_REQ: link-list request mode
127 *
128 * We have 4 types request mode: fragment mode, block mode, transaction mode
129 * and linklist mode. One transaction can contain several blocks, one block can
130 * contain several fragments. Link-list mode means we can save several DMA
131 * configuration into one reserved memory, then DMA can fetch each DMA
132 * configuration automatically to start transfer.
133 */
134enum sprd_dma_req_mode {
135 SPRD_DMA_FRAG_REQ,
136 SPRD_DMA_BLK_REQ,
137 SPRD_DMA_TRANS_REQ,
138 SPRD_DMA_LIST_REQ,
139};
140
141/*
142 * enum sprd_dma_int_type: define the DMA interrupt type
143 * @SPRD_DMA_NO_INT: do not need generate DMA interrupts.
144 * @SPRD_DMA_FRAG_INT: fragment done interrupt when one fragment request
145 * is done.
146 * @SPRD_DMA_BLK_INT: block done interrupt when one block request is done.
147 * @SPRD_DMA_BLK_FRAG_INT: block and fragment interrupt when one fragment
148 * or one block request is done.
149 * @SPRD_DMA_TRANS_INT: tansaction done interrupt when one transaction
150 * request is done.
151 * @SPRD_DMA_TRANS_FRAG_INT: transaction and fragment interrupt when one
152 * transaction request or fragment request is done.
153 * @SPRD_DMA_TRANS_BLK_INT: transaction and block interrupt when one
154 * transaction request or block request is done.
155 * @SPRD_DMA_LIST_INT: link-list done interrupt when one link-list request
156 * is done.
157 * @SPRD_DMA_CFGERR_INT: configure error interrupt when configuration is
158 * incorrect.
159 */
160enum sprd_dma_int_type {
161 SPRD_DMA_NO_INT,
162 SPRD_DMA_FRAG_INT,
163 SPRD_DMA_BLK_INT,
164 SPRD_DMA_BLK_FRAG_INT,
165 SPRD_DMA_TRANS_INT,
166 SPRD_DMA_TRANS_FRAG_INT,
167 SPRD_DMA_TRANS_BLK_INT,
168 SPRD_DMA_LIST_INT,
169 SPRD_DMA_CFGERR_INT,
170};
171
172/* dma channel hardware configuration */
173struct sprd_dma_chn_hw {
174 u32 pause;
175 u32 req;
176 u32 cfg;
177 u32 intc;
178 u32 src_addr;
179 u32 des_addr;
180 u32 frg_len;
181 u32 blk_len;
182 u32 trsc_len;
183 u32 trsf_step;
184 u32 wrap_ptr;
185 u32 wrap_to;
186 u32 llist_ptr;
187 u32 frg_step;
188 u32 src_blk_step;
189 u32 des_blk_step;
190};
191
192/* dma request description */
193struct sprd_dma_desc {
194 struct virt_dma_desc vd;
195 struct sprd_dma_chn_hw chn_hw;
196};
197
198/* dma channel description */
199struct sprd_dma_chn {
200 struct virt_dma_chan vc;
201 void __iomem *chn_base;
202 u32 chn_num;
203 u32 dev_id;
204 struct sprd_dma_desc *cur_desc;
205};
206
207/* SPRD dma device */
208struct sprd_dma_dev {
209 struct dma_device dma_dev;
210 void __iomem *glb_base;
211 struct clk *clk;
212 struct clk *ashb_clk;
213 int irq;
214 u32 total_chns;
215 struct sprd_dma_chn channels[0];
216};
217
71128e8f 218static void sprd_dma_free_desc(struct virt_dma_desc *vd);
9b3b8171
BW		 219static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param);
220static struct of_dma_filter_info sprd_dma_info = {
221 .filter_fn = sprd_dma_filter_fn,
222};
223
224static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c)
225{
226 return container_of(c, struct sprd_dma_chn, vc.chan);
227}
228
229static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c)
230{
231 struct sprd_dma_chn *schan = to_sprd_dma_chan(c);
232
233 return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);
234}
235
236static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd)
237{
238 return container_of(vd, struct sprd_dma_desc, vd);
239}
240
241static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,
242 u32 mask, u32 val)
243{
244 u32 orig = readl(schan->chn_base + reg);
245 u32 tmp;
246
247 tmp = (orig & ~mask) | val;
248 writel(tmp, schan->chn_base + reg);
249}
250
251static int sprd_dma_enable(struct sprd_dma_dev *sdev)
252{
253 int ret;
254
255 ret = clk_prepare_enable(sdev->clk);
256 if (ret)
257 return ret;
258
259 /*
260 * The ashb_clk is optional and only for AGCP DMA controller, so we
261 * need add one condition to check if the ashb_clk need enable.
262 */
263 if (!IS_ERR(sdev->ashb_clk))
264 ret = clk_prepare_enable(sdev->ashb_clk);
265
266 return ret;
267}
268
269static void sprd_dma_disable(struct sprd_dma_dev *sdev)
270{
271 clk_disable_unprepare(sdev->clk);
272
273 /*
274 * Need to check if we need disable the optional ashb_clk for AGCP DMA.
275 */
276 if (!IS_ERR(sdev->ashb_clk))
277 clk_disable_unprepare(sdev->ashb_clk);
278}
279
280static void sprd_dma_set_uid(struct sprd_dma_chn *schan)
281{
282 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
283 u32 dev_id = schan->dev_id;
284
285 if (dev_id != SPRD_DMA_SOFTWARE_UID) {
286 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
287 SPRD_DMA_GLB_REQ_UID(dev_id);
288
289 writel(schan->chn_num + 1, sdev->glb_base + uid_offset);
290 }
291}
292
293static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)
294{
295 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
296 u32 dev_id = schan->dev_id;
297
298 if (dev_id != SPRD_DMA_SOFTWARE_UID) {
299 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
300 SPRD_DMA_GLB_REQ_UID(dev_id);
301
302 writel(0, sdev->glb_base + uid_offset);
303 }
304}
305
306static void sprd_dma_clear_int(struct sprd_dma_chn *schan)
307{
308 sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,
309 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET,
310 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET);
311}
312
313static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)
314{
315 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,
316 SPRD_DMA_CHN_EN);
317}
318
319static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)
320{
321 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);
322}
323
324static void sprd_dma_soft_request(struct sprd_dma_chn *schan)
325{
326 sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,
327 SPRD_DMA_REQ_EN);
328}
329
330static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)
331{
332 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
333 u32 pause, timeout = SPRD_DMA_PAUSE_CNT;
334
335 if (enable) {
336 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
337 SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN);
338
339 do {
340 pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);
341 if (pause & SPRD_DMA_PAUSE_STS)
342 break;
343
344 cpu_relax();
345 } while (--timeout > 0);
346
347 if (!timeout)
348 dev_warn(sdev->dma_dev.dev,
349 "pause dma controller timeout\n");
350 } else {
351 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
352 SPRD_DMA_PAUSE_EN, 0);
353 }
354}
355
356static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)
357{
358 u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);
359
360 if (!(cfg & SPRD_DMA_CHN_EN))
361 return;
362
363 sprd_dma_pause_resume(schan, true);
364 sprd_dma_disable_chn(schan);
365}
366
367static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)
368{
369 unsigned long addr, addr_high;
370
371 addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
372 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &
373 SPRD_DMA_HIGH_ADDR_MASK;
374
375 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
376}
377
378static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)
379{
380 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
381 u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &
382 SPRD_DMA_CHN_INT_STS;
383
384 switch (intc_sts) {
385 case SPRD_DMA_CFGERR_INT_STS:
386 return SPRD_DMA_CFGERR_INT;
387
388 case SPRD_DMA_LIST_INT_STS:
389 return SPRD_DMA_LIST_INT;
390
391 case SPRD_DMA_TRSC_INT_STS:
392 return SPRD_DMA_TRANS_INT;
393
394 case SPRD_DMA_BLK_INT_STS:
395 return SPRD_DMA_BLK_INT;
396
397 case SPRD_DMA_FRAG_INT_STS:
398 return SPRD_DMA_FRAG_INT;
399
400 default:
401 dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n");
402 return SPRD_DMA_NO_INT;
403 }
404}
405
406static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)
407{
408 u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
409
410 return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK;
411}
412
413static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,
414 struct sprd_dma_desc *sdesc)
415{
416 struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw;
417
418 writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);
419 writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);
420 writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);
421 writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
422 writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
423 writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
424 writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);
425 writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);
426 writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);
427 writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);
428 writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);
429 writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);
430 writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);
431 writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);
432 writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);
433 writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);
434}
435
436static void sprd_dma_start(struct sprd_dma_chn *schan)
437{
438 struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);
439
440 if (!vd)
441 return;
442
443 list_del(&vd->node);
444 schan->cur_desc = to_sprd_dma_desc(vd);
445
446 /*
447 * Copy the DMA configuration from DMA descriptor to this hardware
448 * channel.
449 */
450 sprd_dma_set_chn_config(schan, schan->cur_desc);
451 sprd_dma_set_uid(schan);
452 sprd_dma_enable_chn(schan);
453
454 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)
455 sprd_dma_soft_request(schan);
456}
457
458static void sprd_dma_stop(struct sprd_dma_chn *schan)
459{
460 sprd_dma_stop_and_disable(schan);
461 sprd_dma_unset_uid(schan);
462 sprd_dma_clear_int(schan);
463}
464
465static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc,
466 enum sprd_dma_int_type int_type,
467 enum sprd_dma_req_mode req_mode)
468{
469 if (int_type == SPRD_DMA_NO_INT)
470 return false;
471
472 if (int_type >= req_mode + 1)
473 return true;
474 else
475 return false;
476}
477
478static irqreturn_t dma_irq_handle(int irq, void *dev_id)
479{
480 struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id;
481 u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS);
482 struct sprd_dma_chn *schan;
483 struct sprd_dma_desc *sdesc;
484 enum sprd_dma_req_mode req_type;
485 enum sprd_dma_int_type int_type;
486 bool trans_done = false;
487 u32 i;
488
489 while (irq_status) {
490 i = __ffs(irq_status);
491 irq_status &= (irq_status - 1);
492 schan = &sdev->channels[i];
493
494 spin_lock(&schan->vc.lock);
495 int_type = sprd_dma_get_int_type(schan);
496 req_type = sprd_dma_get_req_type(schan);
497 sprd_dma_clear_int(schan);
498
499 sdesc = schan->cur_desc;
500
501 /* Check if the dma request descriptor is done. */
502 trans_done = sprd_dma_check_trans_done(sdesc, int_type,
503 req_type);
504 if (trans_done == true) {
505 vchan_cookie_complete(&sdesc->vd);
506 schan->cur_desc = NULL;
507 sprd_dma_start(schan);
508 }
509 spin_unlock(&schan->vc.lock);
510 }
511
512 return IRQ_HANDLED;
513}
514
515static int sprd_dma_alloc_chan_resources(struct dma_chan *chan)
516{
517 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
518 int ret;
519
520 ret = pm_runtime_get_sync(chan->device->dev);
521 if (ret < 0)
522 return ret;
523
524 schan->dev_id = SPRD_DMA_SOFTWARE_UID;
525 return 0;
526}
527
528static void sprd_dma_free_chan_resources(struct dma_chan *chan)
529{
530 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
71128e8f 531 struct virt_dma_desc *cur_vd = NULL;
9b3b8171
BW		 532 unsigned long flags;
533
534 spin_lock_irqsave(&schan->vc.lock, flags);
71128e8f
BW		 535 if (schan->cur_desc)
536 cur_vd = &schan->cur_desc->vd;
537
9b3b8171
BW		 538 sprd_dma_stop(schan);
539 spin_unlock_irqrestore(&schan->vc.lock, flags);
540
71128e8f
BW		 541 if (cur_vd)
542 sprd_dma_free_desc(cur_vd);
543
9b3b8171
BW		 544 vchan_free_chan_resources(&schan->vc);
545 pm_runtime_put(chan->device->dev);
546}
547
548static enum dma_status sprd_dma_tx_status(struct dma_chan *chan,
549 dma_cookie_t cookie,
550 struct dma_tx_state *txstate)
551{
552 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
553 struct virt_dma_desc *vd;
554 unsigned long flags;
555 enum dma_status ret;
556 u32 pos;
557
558 ret = dma_cookie_status(chan, cookie, txstate);
559 if (ret == DMA_COMPLETE || !txstate)
560 return ret;
561
562 spin_lock_irqsave(&schan->vc.lock, flags);
563 vd = vchan_find_desc(&schan->vc, cookie);
564 if (vd) {
565 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
566 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
567
568 if (hw->trsc_len > 0)
569 pos = hw->trsc_len;
570 else if (hw->blk_len > 0)
571 pos = hw->blk_len;
572 else if (hw->frg_len > 0)
573 pos = hw->frg_len;
574 else
575 pos = 0;
576 } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {
577 pos = sprd_dma_get_dst_addr(schan);
578 } else {
579 pos = 0;
580 }
581 spin_unlock_irqrestore(&schan->vc.lock, flags);
582
583 dma_set_residue(txstate, pos);
584 return ret;
585}
586
587static void sprd_dma_issue_pending(struct dma_chan *chan)
588{
589 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
590 unsigned long flags;
591
592 spin_lock_irqsave(&schan->vc.lock, flags);
593 if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)
594 sprd_dma_start(schan);
595 spin_unlock_irqrestore(&schan->vc.lock, flags);
596}
597
598static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *sdesc,
599 dma_addr_t dest, dma_addr_t src, size_t len)
600{
601 struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
602 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
603 u32 datawidth, src_step, des_step, fragment_len;
604 u32 block_len, req_mode, irq_mode, transcation_len;
605 u32 fix_mode = 0, fix_en = 0;
606
607 if (IS_ALIGNED(len, 4)) {
608 datawidth = 2;
609 src_step = 4;
610 des_step = 4;
611 } else if (IS_ALIGNED(len, 2)) {
612 datawidth = 1;
613 src_step = 2;
614 des_step = 2;
615 } else {
616 datawidth = 0;
617 src_step = 1;
618 des_step = 1;
619 }
620
621 fragment_len = SPRD_DMA_MEMCPY_MIN_SIZE;
622 if (len <= SPRD_DMA_BLK_LEN_MASK) {
623 block_len = len;
624 transcation_len = 0;
625 req_mode = SPRD_DMA_BLK_REQ;
626 irq_mode = SPRD_DMA_BLK_INT;
627 } else {
628 block_len = SPRD_DMA_MEMCPY_MIN_SIZE;
629 transcation_len = len;
630 req_mode = SPRD_DMA_TRANS_REQ;
631 irq_mode = SPRD_DMA_TRANS_INT;
632 }
633
634 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
635 hw->wrap_ptr = (u32)((src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
636 SPRD_DMA_HIGH_ADDR_MASK);
637 hw->wrap_to = (u32)((dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
638 SPRD_DMA_HIGH_ADDR_MASK);
639
640 hw->src_addr = (u32)(src & SPRD_DMA_LOW_ADDR_MASK);
641 hw->des_addr = (u32)(dest & SPRD_DMA_LOW_ADDR_MASK);
642
643 if ((src_step != 0 && des_step != 0) || (src_step | des_step) == 0) {
644 fix_en = 0;
645 } else {
646 fix_en = 1;
647 if (src_step)
648 fix_mode = 1;
649 else
650 fix_mode = 0;
651 }
652
653 hw->frg_len = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET |
654 datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET |
655 req_mode << SPRD_DMA_REQ_MODE_OFFSET |
656 fix_mode << SPRD_DMA_FIX_SEL_OFFSET |
657 fix_en << SPRD_DMA_FIX_EN_OFFSET |
658 (fragment_len & SPRD_DMA_FRG_LEN_MASK);
659 hw->blk_len = block_len & SPRD_DMA_BLK_LEN_MASK;
660
661 hw->intc = SPRD_DMA_CFG_ERR_INT_EN;
662
663 switch (irq_mode) {
664 case SPRD_DMA_NO_INT:
665 break;
666
667 case SPRD_DMA_FRAG_INT:
668 hw->intc |= SPRD_DMA_FRAG_INT_EN;
669 break;
670
671 case SPRD_DMA_BLK_INT:
672 hw->intc |= SPRD_DMA_BLK_INT_EN;
673 break;
674
675 case SPRD_DMA_BLK_FRAG_INT:
676 hw->intc |= SPRD_DMA_BLK_INT_EN | SPRD_DMA_FRAG_INT_EN;
677 break;
678
679 case SPRD_DMA_TRANS_INT:
680 hw->intc |= SPRD_DMA_TRANS_INT_EN;
681 break;
682
683 case SPRD_DMA_TRANS_FRAG_INT:
684 hw->intc |= SPRD_DMA_TRANS_INT_EN | SPRD_DMA_FRAG_INT_EN;
685 break;
686
687 case SPRD_DMA_TRANS_BLK_INT:
688 hw->intc |= SPRD_DMA_TRANS_INT_EN | SPRD_DMA_BLK_INT_EN;
689 break;
690
691 case SPRD_DMA_LIST_INT:
692 hw->intc |= SPRD_DMA_LIST_INT_EN;
693 break;
694
695 case SPRD_DMA_CFGERR_INT:
696 hw->intc |= SPRD_DMA_CFG_ERR_INT_EN;
697 break;
698
699 default:
700 dev_err(sdev->dma_dev.dev, "invalid irq mode\n");
701 return -EINVAL;
702 }
703
704 if (transcation_len == 0)
705 hw->trsc_len = block_len & SPRD_DMA_TRSC_LEN_MASK;
706 else
707 hw->trsc_len = transcation_len & SPRD_DMA_TRSC_LEN_MASK;
708
709 hw->trsf_step = (des_step & SPRD_DMA_TRSF_STEP_MASK) <<
710 SPRD_DMA_DEST_TRSF_STEP_OFFSET |
711 (src_step & SPRD_DMA_TRSF_STEP_MASK) <<
712 SPRD_DMA_SRC_TRSF_STEP_OFFSET;
713
714 hw->frg_step = 0;
715 hw->src_blk_step = 0;
716 hw->des_blk_step = 0;
717 hw->src_blk_step = 0;
718 return 0;
719}
720
721struct dma_async_tx_descriptor *
722sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
723 size_t len, unsigned long flags)
724{
725 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
726 struct sprd_dma_desc *sdesc;
727 int ret;
728
729 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
730 if (!sdesc)
731 return NULL;
732
733 ret = sprd_dma_config(chan, sdesc, dest, src, len);
734 if (ret) {
735 kfree(sdesc);
736 return NULL;
737 }
738
739 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
740}
741
742static int sprd_dma_pause(struct dma_chan *chan)
743{
744 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
745 unsigned long flags;
746
747 spin_lock_irqsave(&schan->vc.lock, flags);
748 sprd_dma_pause_resume(schan, true);
749 spin_unlock_irqrestore(&schan->vc.lock, flags);
750
751 return 0;
752}
753
754static int sprd_dma_resume(struct dma_chan *chan)
755{
756 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
757 unsigned long flags;
758
759 spin_lock_irqsave(&schan->vc.lock, flags);
760 sprd_dma_pause_resume(schan, false);
761 spin_unlock_irqrestore(&schan->vc.lock, flags);
762
763 return 0;
764}
765
766static int sprd_dma_terminate_all(struct dma_chan *chan)
767{
768 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
71128e8f 769 struct virt_dma_desc *cur_vd = NULL;
9b3b8171
BW		 770 unsigned long flags;
771 LIST_HEAD(head);
772
773 spin_lock_irqsave(&schan->vc.lock, flags);
71128e8f
BW		 774 if (schan->cur_desc)
775 cur_vd = &schan->cur_desc->vd;
776
9b3b8171
BW		 777 sprd_dma_stop(schan);
778
779 vchan_get_all_descriptors(&schan->vc, &head);
780 spin_unlock_irqrestore(&schan->vc.lock, flags);
781
71128e8f
BW		 782 if (cur_vd)
783 sprd_dma_free_desc(cur_vd);
784
9b3b8171
BW		 785 vchan_dma_desc_free_list(&schan->vc, &head);
786 return 0;
787}
788
789static void sprd_dma_free_desc(struct virt_dma_desc *vd)
790{
791 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
792
793 kfree(sdesc);
794}
795
796static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param)
797{
798 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
799 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
800 u32 req = *(u32 *)param;
801
802 if (req < sdev->total_chns)
803 return req == schan->chn_num + 1;
804 else
805 return false;
806}
807
808static int sprd_dma_probe(struct platform_device *pdev)
809{
810 struct device_node *np = pdev->dev.of_node;
811 struct sprd_dma_dev *sdev;
812 struct sprd_dma_chn *dma_chn;
813 struct resource *res;
814 u32 chn_count;
815 int ret, i;
816
817 ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count);
818 if (ret) {
819 dev_err(&pdev->dev, "get dma channels count failed\n");
820 return ret;
821 }
822
823 sdev = devm_kzalloc(&pdev->dev, sizeof(*sdev) +
824 sizeof(*dma_chn) * chn_count,
825 GFP_KERNEL);
826 if (!sdev)
827 return -ENOMEM;
828
829 sdev->clk = devm_clk_get(&pdev->dev, "enable");
830 if (IS_ERR(sdev->clk)) {
831 dev_err(&pdev->dev, "get enable clock failed\n");
832 return PTR_ERR(sdev->clk);
833 }
834
835 /* ashb clock is optional for AGCP DMA */
836 sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
837 if (IS_ERR(sdev->ashb_clk))
838 dev_warn(&pdev->dev, "no optional ashb eb clock\n");
839
840 /*
841 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP
842 * DMA controller, it can or do not request the irq, which will save
843 * system power without resuming system by DMA interrupts if AGCP DMA
844 * does not request the irq. Thus the DMA interrupts property should
845 * be optional.
846 */
847 sdev->irq = platform_get_irq(pdev, 0);
848 if (sdev->irq > 0) {
849 ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle,
850 0, "sprd_dma", (void *)sdev);
851 if (ret < 0) {
852 dev_err(&pdev->dev, "request dma irq failed\n");
853 return ret;
854 }
855 } else {
856 dev_warn(&pdev->dev, "no interrupts for the dma controller\n");
857 }
858
859 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
860 sdev->glb_base = devm_ioremap_nocache(&pdev->dev, res->start,
861 resource_size(res));
862 if (!sdev->glb_base)
863 return -ENOMEM;
864
865 dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
866 sdev->total_chns = chn_count;
867 sdev->dma_dev.chancnt = chn_count;
868 INIT_LIST_HEAD(&sdev->dma_dev.channels);
869 INIT_LIST_HEAD(&sdev->dma_dev.global_node);
870 sdev->dma_dev.dev = &pdev->dev;
871 sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources;
872 sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources;
873 sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
874 sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
875 sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
876 sdev->dma_dev.device_pause = sprd_dma_pause;
877 sdev->dma_dev.device_resume = sprd_dma_resume;
878 sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all;
879
880 for (i = 0; i < chn_count; i++) {
881 dma_chn = &sdev->channels[i];
882 dma_chn->chn_num = i;
883 dma_chn->cur_desc = NULL;
884 /* get each channel's registers base address. */
885 dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET +
886 SPRD_DMA_CHN_REG_LENGTH * i;
887
888 dma_chn->vc.desc_free = sprd_dma_free_desc;
889 vchan_init(&dma_chn->vc, &sdev->dma_dev);
890 }
891
892 platform_set_drvdata(pdev, sdev);
893 ret = sprd_dma_enable(sdev);
894 if (ret)
895 return ret;
896
897 pm_runtime_set_active(&pdev->dev);
898 pm_runtime_enable(&pdev->dev);
899
900 ret = pm_runtime_get_sync(&pdev->dev);
901 if (ret < 0)
902 goto err_rpm;
903
904 ret = dma_async_device_register(&sdev->dma_dev);
905 if (ret < 0) {
906 dev_err(&pdev->dev, "register dma device failed:%d\n", ret);
907 goto err_register;
908 }
909
910 sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask;
911 ret = of_dma_controller_register(np, of_dma_simple_xlate,
912 &sprd_dma_info);
913 if (ret)
914 goto err_of_register;
915
916 pm_runtime_put(&pdev->dev);
917 return 0;
918
919err_of_register:
920 dma_async_device_unregister(&sdev->dma_dev);
921err_register:
922 pm_runtime_put_noidle(&pdev->dev);
923 pm_runtime_disable(&pdev->dev);
924err_rpm:
925 sprd_dma_disable(sdev);
926 return ret;
927}
928
929static int sprd_dma_remove(struct platform_device *pdev)
930{
931 struct sprd_dma_dev *sdev = platform_get_drvdata(pdev);
932 struct sprd_dma_chn *c, *cn;
933 int ret;
934
935 ret = pm_runtime_get_sync(&pdev->dev);
936 if (ret < 0)
937 return ret;
938
939 /* explicitly free the irq */
940 if (sdev->irq > 0)
941 devm_free_irq(&pdev->dev, sdev->irq, sdev);
942
943 list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels,
944 vc.chan.device_node) {
945 list_del(&c->vc.chan.device_node);
946 tasklet_kill(&c->vc.task);
947 }
948
949 of_dma_controller_free(pdev->dev.of_node);
950 dma_async_device_unregister(&sdev->dma_dev);
951 sprd_dma_disable(sdev);
952
953 pm_runtime_put_noidle(&pdev->dev);
954 pm_runtime_disable(&pdev->dev);
955 return 0;
956}
957
958static const struct of_device_id sprd_dma_match[] = {
959 { .compatible = "sprd,sc9860-dma", },
960 {},
961};
962
963static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
964{
965 struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
966
967 sprd_dma_disable(sdev);
968 return 0;
969}
970
971static int __maybe_unused sprd_dma_runtime_resume(struct device *dev)
972{
973 struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
974 int ret;
975
976 ret = sprd_dma_enable(sdev);
977 if (ret)
978 dev_err(sdev->dma_dev.dev, "enable dma failed\n");
979
980 return ret;
981}
982
983static const struct dev_pm_ops sprd_dma_pm_ops = {
984 SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend,
985 sprd_dma_runtime_resume,
986 NULL)
987};
988
989static struct platform_driver sprd_dma_driver = {
990 .probe = sprd_dma_probe,
991 .remove = sprd_dma_remove,
992 .driver = {
993 .name = "sprd-dma",
994 .of_match_table = sprd_dma_match,
995 .pm = &sprd_dma_pm_ops,
996 },
997};
998module_platform_driver(sprd_dma_driver);
999
1000MODULE_LICENSE("GPL v2");
1001MODULE_DESCRIPTION("DMA driver for Spreadtrum");
1002MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
1003MODULE_ALIAS("platform:sprd-dma");
ubuntu-bionic-kernel mirror