]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/dma/ste_dma40.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ARM: ux500: Strip out duplicate USB DMA configuration
[mirror_ubuntu-bionic-kernel.git] / drivers / dma / ste_dma40.c
CommitLineData
8d318a50 1/*
d49278e3
PF		 2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
661385f9 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
767a9675 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
8d318a50 6 * License terms: GNU General Public License (GPL) version 2
8d318a50
LW		 7 */
8
b7f080cf 9#include <linux/dma-mapping.h>
8d318a50
LW		 10#include <linux/kernel.h>
11#include <linux/slab.h>
f492b210 12#include <linux/export.h>
8d318a50
LW		 13#include <linux/dmaengine.h>
14#include <linux/platform_device.h>
15#include <linux/clk.h>
16#include <linux/delay.h>
7fb3e75e
N		 17#include <linux/pm.h>
18#include <linux/pm_runtime.h>
698e4732 19#include <linux/err.h>
f4b89764 20#include <linux/amba/bus.h>
15e4b78d 21#include <linux/regulator/consumer.h>
865fab60 22#include <linux/platform_data/dma-ste-dma40.h>
8d318a50 23
d2ebfb33 24#include "dmaengine.h"
8d318a50
LW		 25#include "ste_dma40_ll.h"
26
27#define D40_NAME "dma40"
28
29#define D40_PHY_CHAN -1
30
31/* For masking out/in 2 bit channel positions */
32#define D40_CHAN_POS(chan) (2 * (chan / 2))
33#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
34
35/* Maximum iterations taken before giving up suspending a channel */
36#define D40_SUSPEND_MAX_IT 500
37
7fb3e75e
N		 38/* Milliseconds */
39#define DMA40_AUTOSUSPEND_DELAY 100
40
508849ad
LW		 41/* Hardware requirement on LCLA alignment */
42#define LCLA_ALIGNMENT 0x40000
698e4732
JA		 43
44/* Max number of links per event group */
45#define D40_LCLA_LINK_PER_EVENT_GRP 128
46#define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
47
508849ad
LW		 48/* Attempts before giving up to trying to get pages that are aligned */
49#define MAX_LCLA_ALLOC_ATTEMPTS 256
50
51/* Bit markings for allocation map */
8d318a50
LW		 52#define D40_ALLOC_FREE (1 << 31)
53#define D40_ALLOC_PHY (1 << 30)
54#define D40_ALLOC_LOG_FREE 0
55
3cb645dc
TL		 56#define MAX(a, b) (((a) < (b)) ? (b) : (a))
57
664a57ec 58/* Reserved event lines for memcpy only. */
a2acaa21
LW		 59#define DB8500_DMA_MEMCPY_EV_0 51
60#define DB8500_DMA_MEMCPY_EV_1 56
61#define DB8500_DMA_MEMCPY_EV_2 57
62#define DB8500_DMA_MEMCPY_EV_3 58
63#define DB8500_DMA_MEMCPY_EV_4 59
64#define DB8500_DMA_MEMCPY_EV_5 60
65
66static int dma40_memcpy_channels[] = {
67 DB8500_DMA_MEMCPY_EV_0,
68 DB8500_DMA_MEMCPY_EV_1,
69 DB8500_DMA_MEMCPY_EV_2,
70 DB8500_DMA_MEMCPY_EV_3,
71 DB8500_DMA_MEMCPY_EV_4,
72 DB8500_DMA_MEMCPY_EV_5,
73};
664a57ec 74
29027a1e
LJ		 75/* Default configuration for physcial memcpy */
76struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
77 .mode = STEDMA40_MODE_PHYSICAL,
78 .dir = STEDMA40_MEM_TO_MEM,
79
80 .src_info.data_width = STEDMA40_BYTE_WIDTH,
81 .src_info.psize = STEDMA40_PSIZE_PHY_1,
82 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
83
84 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
85 .dst_info.psize = STEDMA40_PSIZE_PHY_1,
86 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
87};
88
89/* Default configuration for logical memcpy */
90struct stedma40_chan_cfg dma40_memcpy_conf_log = {
91 .mode = STEDMA40_MODE_LOGICAL,
92 .dir = STEDMA40_MEM_TO_MEM,
93
94 .src_info.data_width = STEDMA40_BYTE_WIDTH,
95 .src_info.psize = STEDMA40_PSIZE_LOG_1,
96 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
97
98 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
99 .dst_info.psize = STEDMA40_PSIZE_LOG_1,
100 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
101};
102
8d318a50
LW		 103/**
104 * enum 40_command - The different commands and/or statuses.
105 *
106 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
107 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
108 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
109 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
110 */
111enum d40_command {
112 D40_DMA_STOP = 0,
113 D40_DMA_RUN = 1,
114 D40_DMA_SUSPEND_REQ = 2,
115 D40_DMA_SUSPENDED = 3
116};
117
1bdae6f4
N		 118/*
119 * enum d40_events - The different Event Enables for the event lines.
120 *
121 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
122 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
123 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
124 * @D40_ROUND_EVENTLINE: Status check for event line.
125 */
126
127enum d40_events {
128 D40_DEACTIVATE_EVENTLINE = 0,
129 D40_ACTIVATE_EVENTLINE = 1,
130 D40_SUSPEND_REQ_EVENTLINE = 2,
131 D40_ROUND_EVENTLINE = 3
132};
133
7fb3e75e
N		 134/*
135 * These are the registers that has to be saved and later restored
136 * when the DMA hw is powered off.
137 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
138 */
139static u32 d40_backup_regs[] = {
140 D40_DREG_LCPA,
141 D40_DREG_LCLA,
142 D40_DREG_PRMSE,
143 D40_DREG_PRMSO,
144 D40_DREG_PRMOE,
145 D40_DREG_PRMOO,
146};
147
148#define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
149
3cb645dc
TL		 150/*
151 * since 9540 and 8540 has the same HW revision
152 * use v4a for 9540 or ealier
153 * use v4b for 8540 or later
154 * HW revision:
155 * DB8500ed has revision 0
156 * DB8500v1 has revision 2
157 * DB8500v2 has revision 3
158 * AP9540v1 has revision 4
159 * DB8540v1 has revision 4
160 * TODO: Check if all these registers have to be saved/restored on dma40 v4a
161 */
162static u32 d40_backup_regs_v4a[] = {
7fb3e75e
N		 163 D40_DREG_PSEG1,
164 D40_DREG_PSEG2,
165 D40_DREG_PSEG3,
166 D40_DREG_PSEG4,
167 D40_DREG_PCEG1,
168 D40_DREG_PCEG2,
169 D40_DREG_PCEG3,
170 D40_DREG_PCEG4,
171 D40_DREG_RSEG1,
172 D40_DREG_RSEG2,
173 D40_DREG_RSEG3,
174 D40_DREG_RSEG4,
175 D40_DREG_RCEG1,
176 D40_DREG_RCEG2,
177 D40_DREG_RCEG3,
178 D40_DREG_RCEG4,
179};
180
3cb645dc
TL		 181#define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
182
183static u32 d40_backup_regs_v4b[] = {
184 D40_DREG_CPSEG1,
185 D40_DREG_CPSEG2,
186 D40_DREG_CPSEG3,
187 D40_DREG_CPSEG4,
188 D40_DREG_CPSEG5,
189 D40_DREG_CPCEG1,
190 D40_DREG_CPCEG2,
191 D40_DREG_CPCEG3,
192 D40_DREG_CPCEG4,
193 D40_DREG_CPCEG5,
194 D40_DREG_CRSEG1,
195 D40_DREG_CRSEG2,
196 D40_DREG_CRSEG3,
197 D40_DREG_CRSEG4,
198 D40_DREG_CRSEG5,
199 D40_DREG_CRCEG1,
200 D40_DREG_CRCEG2,
201 D40_DREG_CRCEG3,
202 D40_DREG_CRCEG4,
203 D40_DREG_CRCEG5,
204};
205
206#define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
7fb3e75e
N		 207
208static u32 d40_backup_regs_chan[] = {
209 D40_CHAN_REG_SSCFG,
210 D40_CHAN_REG_SSELT,
211 D40_CHAN_REG_SSPTR,
212 D40_CHAN_REG_SSLNK,
213 D40_CHAN_REG_SDCFG,
214 D40_CHAN_REG_SDELT,
215 D40_CHAN_REG_SDPTR,
216 D40_CHAN_REG_SDLNK,
217};
218
3cb645dc
TL		 219/**
220 * struct d40_interrupt_lookup - lookup table for interrupt handler
221 *
222 * @src: Interrupt mask register.
223 * @clr: Interrupt clear register.
224 * @is_error: true if this is an error interrupt.
225 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
226 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
227 */
228struct d40_interrupt_lookup {
229 u32 src;
230 u32 clr;
231 bool is_error;
232 int offset;
233};
234
235
236static struct d40_interrupt_lookup il_v4a[] = {
237 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
238 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
239 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
240 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
241 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
242 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
243 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
244 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
245 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
246 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
247};
248
249static struct d40_interrupt_lookup il_v4b[] = {
250 {D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false, 0},
251 {D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},
252 {D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},
253 {D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},
254 {D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},
255 {D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true, 0},
256 {D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true, 32},
257 {D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true, 64},
258 {D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true, 96},
259 {D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true, 128},
260 {D40_DREG_CPCTIS, D40_DREG_CPCICR, false, D40_PHY_CHAN},
261 {D40_DREG_CPCEIS, D40_DREG_CPCICR, true, D40_PHY_CHAN},
262};
263
264/**
265 * struct d40_reg_val - simple lookup struct
266 *
267 * @reg: The register.
268 * @val: The value that belongs to the register in reg.
269 */
270struct d40_reg_val {
271 unsigned int reg;
272 unsigned int val;
273};
274
275static __initdata struct d40_reg_val dma_init_reg_v4a[] = {
276 /* Clock every part of the DMA block from start */
277 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
278
279 /* Interrupts on all logical channels */
280 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
281 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
282 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
283 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
284 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
285 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
286 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
287 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
288 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
289 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
290 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
291 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
292};
293static __initdata struct d40_reg_val dma_init_reg_v4b[] = {
294 /* Clock every part of the DMA block from start */
295 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
296
297 /* Interrupts on all logical channels */
298 { .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},
299 { .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},
300 { .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},
301 { .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},
302 { .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},
303 { .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},
304 { .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},
305 { .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},
306 { .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},
307 { .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},
308 { .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},
309 { .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},
310 { .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},
311 { .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},
312 { .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}
313};
314
8d318a50
LW		 315/**
316 * struct d40_lli_pool - Structure for keeping LLIs in memory
317 *
318 * @base: Pointer to memory area when the pre_alloc_lli's are not large
319 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
320 * pre_alloc_lli is used.
b00f938c 321 * @dma_addr: DMA address, if mapped
8d318a50
LW		 322 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
323 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
324 * one buffer to one buffer.
325 */
326struct d40_lli_pool {
327 void *base;
508849ad 328 int size;
b00f938c 329 dma_addr_t dma_addr;
8d318a50 330 /* Space for dst and src, plus an extra for padding */
508849ad 331 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
8d318a50
LW		 332};
333
334/**
335 * struct d40_desc - A descriptor is one DMA job.
336 *
337 * @lli_phy: LLI settings for physical channel. Both src and dst=
338 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
339 * lli_len equals one.
340 * @lli_log: Same as above but for logical channels.
341 * @lli_pool: The pool with two entries pre-allocated.
941b77a3 342 * @lli_len: Number of llis of current descriptor.
25985edc 343 * @lli_current: Number of transferred llis.
698e4732 344 * @lcla_alloc: Number of LCLA entries allocated.
8d318a50
LW		 345 * @txd: DMA engine struct. Used for among other things for communication
346 * during a transfer.
347 * @node: List entry.
8d318a50 348 * @is_in_client_list: true if the client owns this descriptor.
7fb3e75e 349 * @cyclic: true if this is a cyclic job
8d318a50
LW		 350 *
351 * This descriptor is used for both logical and physical transfers.
352 */
8d318a50
LW		 353struct d40_desc {
354 /* LLI physical */
355 struct d40_phy_lli_bidir lli_phy;
356 /* LLI logical */
357 struct d40_log_lli_bidir lli_log;
358
359 struct d40_lli_pool lli_pool;
941b77a3 360 int lli_len;
698e4732
JA		 361 int lli_current;
362 int lcla_alloc;
8d318a50
LW		 363
364 struct dma_async_tx_descriptor txd;
365 struct list_head node;
366
8d318a50 367 bool is_in_client_list;
0c842b55 368 bool cyclic;
8d318a50
LW		 369};
370
371/**
372 * struct d40_lcla_pool - LCLA pool settings and data.
373 *
508849ad
LW		 374 * @base: The virtual address of LCLA. 18 bit aligned.
375 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
376 * This pointer is only there for clean-up on error.
377 * @pages: The number of pages needed for all physical channels.
378 * Only used later for clean-up on error
8d318a50 379 * @lock: Lock to protect the content in this struct.
698e4732 380 * @alloc_map: big map over which LCLA entry is own by which job.
8d318a50
LW		 381 */
382struct d40_lcla_pool {
383 void *base;
026cbc42 384 dma_addr_t dma_addr;
508849ad
LW		 385 void *base_unaligned;
386 int pages;
8d318a50 387 spinlock_t lock;
698e4732 388 struct d40_desc **alloc_map;
8d318a50
LW		 389};
390
391/**
392 * struct d40_phy_res - struct for handling eventlines mapped to physical
393 * channels.
394 *
395 * @lock: A lock protection this entity.
7fb3e75e 396 * @reserved: True if used by secure world or otherwise.
8d318a50
LW		 397 * @num: The physical channel number of this entity.
398 * @allocated_src: Bit mapped to show which src event line's are mapped to
399 * this physical channel. Can also be free or physically allocated.
400 * @allocated_dst: Same as for src but is dst.
401 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
767a9675 402 * event line number.
7407048b 403 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
8d318a50
LW		 404 */
405struct d40_phy_res {
406 spinlock_t lock;
7fb3e75e 407 bool reserved;
8d318a50
LW		 408 int num;
409 u32 allocated_src;
410 u32 allocated_dst;
7407048b 411 bool use_soft_lli;
8d318a50
LW		 412};
413
414struct d40_base;
415
416/**
417 * struct d40_chan - Struct that describes a channel.
418 *
419 * @lock: A spinlock to protect this struct.
420 * @log_num: The logical number, if any of this channel.
8d318a50
LW		 421 * @pending_tx: The number of pending transfers. Used between interrupt handler
422 * and tasklet.
423 * @busy: Set to true when transfer is ongoing on this channel.
2a614340
JA		 424 * @phy_chan: Pointer to physical channel which this instance runs on. If this
425 * point is NULL, then the channel is not allocated.
8d318a50
LW		 426 * @chan: DMA engine handle.
427 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
428 * transfer and call client callback.
429 * @client: Cliented owned descriptor list.
da063d26 430 * @pending_queue: Submitted jobs, to be issued by issue_pending()
8d318a50 431 * @active: Active descriptor.
4226dd86 432 * @done: Completed jobs
8d318a50 433 * @queue: Queued jobs.
82babbb3 434 * @prepare_queue: Prepared jobs.
8d318a50 435 * @dma_cfg: The client configuration of this dma channel.
ce2ca125 436 * @configured: whether the dma_cfg configuration is valid
8d318a50
LW		 437 * @base: Pointer to the device instance struct.
438 * @src_def_cfg: Default cfg register setting for src.
439 * @dst_def_cfg: Default cfg register setting for dst.
440 * @log_def: Default logical channel settings.
8d318a50 441 * @lcpa: Pointer to dst and src lcpa settings.
ae752bf4 442 * @runtime_addr: runtime configured address.
443 * @runtime_direction: runtime configured direction.
8d318a50
LW		 444 *
445 * This struct can either "be" a logical or a physical channel.
446 */
447struct d40_chan {
448 spinlock_t lock;
449 int log_num;
8d318a50
LW		 450 int pending_tx;
451 bool busy;
452 struct d40_phy_res *phy_chan;
453 struct dma_chan chan;
454 struct tasklet_struct tasklet;
455 struct list_head client;
a8f3067b 456 struct list_head pending_queue;
8d318a50 457 struct list_head active;
4226dd86 458 struct list_head done;
8d318a50 459 struct list_head queue;
82babbb3 460 struct list_head prepare_queue;
8d318a50 461 struct stedma40_chan_cfg dma_cfg;
ce2ca125 462 bool configured;
8d318a50
LW		 463 struct d40_base *base;
464 /* Default register configurations */
465 u32 src_def_cfg;
466 u32 dst_def_cfg;
467 struct d40_def_lcsp log_def;
8d318a50 468 struct d40_log_lli_full *lcpa;
95e1400f
LW		 469 /* Runtime reconfiguration */
470 dma_addr_t runtime_addr;
db8196df 471 enum dma_transfer_direction runtime_direction;
8d318a50
LW		 472};
473
3cb645dc
TL		 474/**
475 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
476 * controller
477 *
478 * @backup: the pointer to the registers address array for backup
479 * @backup_size: the size of the registers address array for backup
480 * @realtime_en: the realtime enable register
481 * @realtime_clear: the realtime clear register
482 * @high_prio_en: the high priority enable register
483 * @high_prio_clear: the high priority clear register
484 * @interrupt_en: the interrupt enable register
485 * @interrupt_clear: the interrupt clear register
486 * @il: the pointer to struct d40_interrupt_lookup
487 * @il_size: the size of d40_interrupt_lookup array
488 * @init_reg: the pointer to the struct d40_reg_val
489 * @init_reg_size: the size of d40_reg_val array
490 */
491struct d40_gen_dmac {
492 u32 *backup;
493 u32 backup_size;
494 u32 realtime_en;
495 u32 realtime_clear;
496 u32 high_prio_en;
497 u32 high_prio_clear;
498 u32 interrupt_en;
499 u32 interrupt_clear;
500 struct d40_interrupt_lookup *il;
501 u32 il_size;
502 struct d40_reg_val *init_reg;
503 u32 init_reg_size;
504};
505
8d318a50
LW		 506/**
507 * struct d40_base - The big global struct, one for each probe'd instance.
508 *
509 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
510 * @execmd_lock: Lock for execute command usage since several channels share
511 * the same physical register.
512 * @dev: The device structure.
513 * @virtbase: The virtual base address of the DMA's register.
f4185592 514 * @rev: silicon revision detected.
8d318a50
LW		 515 * @clk: Pointer to the DMA clock structure.
516 * @phy_start: Physical memory start of the DMA registers.
517 * @phy_size: Size of the DMA register map.
518 * @irq: The IRQ number.
519 * @num_phy_chans: The number of physical channels. Read from HW. This
520 * is the number of available channels for this driver, not counting "Secure
521 * mode" allocated physical channels.
522 * @num_log_chans: The number of logical channels. Calculated from
523 * num_phy_chans.
524 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
525 * @dma_slave: dma_device channels that can do only do slave transfers.
526 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
7fb3e75e 527 * @phy_chans: Room for all possible physical channels in system.
8d318a50
LW		 528 * @log_chans: Room for all possible logical channels in system.
529 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
530 * to log_chans entries.
531 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
532 * to phy_chans entries.
533 * @plat_data: Pointer to provided platform_data which is the driver
534 * configuration.
28c7a19d 535 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
8d318a50
LW		 536 * @phy_res: Vector containing all physical channels.
537 * @lcla_pool: lcla pool settings and data.
538 * @lcpa_base: The virtual mapped address of LCPA.
539 * @phy_lcpa: The physical address of the LCPA.
540 * @lcpa_size: The size of the LCPA area.
c675b1b4 541 * @desc_slab: cache for descriptors.
7fb3e75e
N		 542 * @reg_val_backup: Here the values of some hardware registers are stored
543 * before the DMA is powered off. They are restored when the power is back on.
3cb645dc
TL		 544 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
545 * later
7fb3e75e
N		 546 * @reg_val_backup_chan: Backup data for standard channel parameter registers.
547 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
548 * @initialized: true if the dma has been initialized
3cb645dc
TL		 549 * @gen_dmac: the struct for generic registers values to represent u8500/8540
550 * DMA controller
8d318a50
LW		 551 */
552struct d40_base {
553 spinlock_t interrupt_lock;
554 spinlock_t execmd_lock;
555 struct device *dev;
556 void __iomem *virtbase;
f4185592 557 u8 rev:4;
8d318a50
LW		 558 struct clk *clk;
559 phys_addr_t phy_start;
560 resource_size_t phy_size;
561 int irq;
562 int num_phy_chans;
563 int num_log_chans;
b96710e5 564 struct device_dma_parameters dma_parms;
8d318a50
LW		 565 struct dma_device dma_both;
566 struct dma_device dma_slave;
567 struct dma_device dma_memcpy;
568 struct d40_chan *phy_chans;
569 struct d40_chan *log_chans;
570 struct d40_chan **lookup_log_chans;
571 struct d40_chan **lookup_phy_chans;
572 struct stedma40_platform_data *plat_data;
28c7a19d 573 struct regulator *lcpa_regulator;
8d318a50
LW		 574 /* Physical half channels */
575 struct d40_phy_res *phy_res;
576 struct d40_lcla_pool lcla_pool;
577 void *lcpa_base;
578 dma_addr_t phy_lcpa;
579 resource_size_t lcpa_size;
c675b1b4 580 struct kmem_cache *desc_slab;
7fb3e75e 581 u32 reg_val_backup[BACKUP_REGS_SZ];
3cb645dc 582 u32 reg_val_backup_v4[MAX(BACKUP_REGS_SZ_V4A, BACKUP_REGS_SZ_V4B)];
7fb3e75e
N		 583 u32 *reg_val_backup_chan;
584 u16 gcc_pwr_off_mask;
585 bool initialized;
3cb645dc 586 struct d40_gen_dmac gen_dmac;
8d318a50
LW		 587};
588
262d2915
RV		 589static struct device *chan2dev(struct d40_chan *d40c)
590{
591 return &d40c->chan.dev->device;
592}
593
724a8577
RV		 594static bool chan_is_physical(struct d40_chan *chan)
595{
596 return chan->log_num == D40_PHY_CHAN;
597}
598
599static bool chan_is_logical(struct d40_chan *chan)
600{
601 return !chan_is_physical(chan);
602}
603
8ca84687
RV		 604static void __iomem *chan_base(struct d40_chan *chan)
605{
606 return chan->base->virtbase + D40_DREG_PCBASE +
607 chan->phy_chan->num * D40_DREG_PCDELTA;
608}
609
6db5a8ba
RV		 610#define d40_err(dev, format, arg...) \
611 dev_err(dev, "[%s] " format, __func__, ## arg)
612
613#define chan_err(d40c, format, arg...) \
614 d40_err(chan2dev(d40c), format, ## arg)
615
b00f938c 616static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
dbd88788 617 int lli_len)
8d318a50 618{
dbd88788 619 bool is_log = chan_is_logical(d40c);
8d318a50
LW		 620 u32 align;
621 void *base;
622
623 if (is_log)
624 align = sizeof(struct d40_log_lli);
625 else
626 align = sizeof(struct d40_phy_lli);
627
628 if (lli_len == 1) {
629 base = d40d->lli_pool.pre_alloc_lli;
630 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
631 d40d->lli_pool.base = NULL;
632 } else {
594ece4d 633 d40d->lli_pool.size = lli_len * 2 * align;
8d318a50
LW		 634
635 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
636 d40d->lli_pool.base = base;
637
638 if (d40d->lli_pool.base == NULL)
639 return -ENOMEM;
640 }
641
642 if (is_log) {
d924abad 643 d40d->lli_log.src = PTR_ALIGN(base, align);
594ece4d 644 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
b00f938c
RV		 645
646 d40d->lli_pool.dma_addr = 0;
8d318a50 647 } else {
d924abad 648 d40d->lli_phy.src = PTR_ALIGN(base, align);
594ece4d 649 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
b00f938c
RV		 650
651 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
652 d40d->lli_phy.src,
653 d40d->lli_pool.size,
654 DMA_TO_DEVICE);
655
656 if (dma_mapping_error(d40c->base->dev,
657 d40d->lli_pool.dma_addr)) {
658 kfree(d40d->lli_pool.base);
659 d40d->lli_pool.base = NULL;
660 d40d->lli_pool.dma_addr = 0;
661 return -ENOMEM;
662 }
8d318a50
LW		 663 }
664
665 return 0;
666}
667
b00f938c 668static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
8d318a50 669{
b00f938c
RV		 670 if (d40d->lli_pool.dma_addr)
671 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
672 d40d->lli_pool.size, DMA_TO_DEVICE);
673
8d318a50
LW		 674 kfree(d40d->lli_pool.base);
675 d40d->lli_pool.base = NULL;
676 d40d->lli_pool.size = 0;
677 d40d->lli_log.src = NULL;
678 d40d->lli_log.dst = NULL;
679 d40d->lli_phy.src = NULL;
680 d40d->lli_phy.dst = NULL;
8d318a50
LW		 681}
682
698e4732
JA		 683static int d40_lcla_alloc_one(struct d40_chan *d40c,
684 struct d40_desc *d40d)
685{
686 unsigned long flags;
687 int i;
688 int ret = -EINVAL;
698e4732
JA		 689
690 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
691
698e4732
JA		 692 /*
693 * Allocate both src and dst at the same time, therefore the half
694 * start on 1 since 0 can't be used since zero is used as end marker.
695 */
696 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
7ce529ef
FB		 697 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
698
699 if (!d40c->base->lcla_pool.alloc_map[idx]) {
700 d40c->base->lcla_pool.alloc_map[idx] = d40d;
698e4732
JA		 701 d40d->lcla_alloc++;
702 ret = i;
703 break;
704 }
705 }
706
707 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
708
709 return ret;
710}
711
712static int d40_lcla_free_all(struct d40_chan *d40c,
713 struct d40_desc *d40d)
714{
715 unsigned long flags;
716 int i;
717 int ret = -EINVAL;
718
724a8577 719 if (chan_is_physical(d40c))
698e4732
JA		 720 return 0;
721
722 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
723
724 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
7ce529ef
FB		 725 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
726
727 if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {
728 d40c->base->lcla_pool.alloc_map[idx] = NULL;
698e4732
JA		 729 d40d->lcla_alloc--;
730 if (d40d->lcla_alloc == 0) {
731 ret = 0;
732 break;
733 }
734 }
735 }
736
737 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
738
739 return ret;
740
741}
742
8d318a50
LW		 743static void d40_desc_remove(struct d40_desc *d40d)
744{
745 list_del(&d40d->node);
746}
747
748static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
749{
a2c15fa4 750 struct d40_desc *desc = NULL;
8d318a50
LW		 751
752 if (!list_empty(&d40c->client)) {
a2c15fa4
RV		 753 struct d40_desc *d;
754 struct d40_desc *_d;
755
7fb3e75e 756 list_for_each_entry_safe(d, _d, &d40c->client, node) {
8d318a50 757 if (async_tx_test_ack(&d->txd)) {
8d318a50 758 d40_desc_remove(d);
a2c15fa4
RV		 759 desc = d;
760 memset(desc, 0, sizeof(*desc));
c675b1b4 761 break;
8d318a50 762 }
7fb3e75e 763 }
8d318a50 764 }
a2c15fa4
RV		 765
766 if (!desc)
767 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
768
769 if (desc)
770 INIT_LIST_HEAD(&desc->node);
771
772 return desc;
8d318a50
LW		 773}
774
775static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
776{
698e4732 777
b00f938c 778 d40_pool_lli_free(d40c, d40d);
698e4732 779 d40_lcla_free_all(d40c, d40d);
c675b1b4 780 kmem_cache_free(d40c->base->desc_slab, d40d);
8d318a50
LW		 781}
782
783static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
784{
785 list_add_tail(&desc->node, &d40c->active);
786}
787
1c4b0927
RV		 788static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
789{
790 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
791 struct d40_phy_lli *lli_src = desc->lli_phy.src;
792 void __iomem *base = chan_base(chan);
793
794 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
795 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
796 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
797 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
798
799 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
800 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
801 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
802 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
803}
804
4226dd86
FB		 805static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)
806{
807 list_add_tail(&desc->node, &d40c->done);
808}
809
e65889c7 810static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
698e4732 811{
e65889c7
RV		 812 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
813 struct d40_log_lli_bidir *lli = &desc->lli_log;
814 int lli_current = desc->lli_current;
815 int lli_len = desc->lli_len;
0c842b55 816 bool cyclic = desc->cyclic;
e65889c7 817 int curr_lcla = -EINVAL;
0c842b55 818 int first_lcla = 0;
28c7a19d 819 bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
0c842b55 820 bool linkback;
e65889c7 821
0c842b55
RV		 822 /*
823 * We may have partially running cyclic transfers, in case we did't get
824 * enough LCLA entries.
825 */
826 linkback = cyclic && lli_current == 0;
827
828 /*
829 * For linkback, we need one LCLA even with only one link, because we
830 * can't link back to the one in LCPA space
831 */
832 if (linkback || (lli_len - lli_current > 1)) {
7407048b
FB		 833 /*
834 * If the channel is expected to use only soft_lli don't
835 * allocate a lcla. This is to avoid a HW issue that exists
836 * in some controller during a peripheral to memory transfer
837 * that uses linked lists.
838 */
839 if (!(chan->phy_chan->use_soft_lli &&
840 chan->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM))
841 curr_lcla = d40_lcla_alloc_one(chan, desc);
842
0c842b55
RV		 843 first_lcla = curr_lcla;
844 }
845
846 /*
847 * For linkback, we normally load the LCPA in the loop since we need to
848 * link it to the second LCLA and not the first. However, if we
849 * couldn't even get a first LCLA, then we have to run in LCPA and
850 * reload manually.
851 */
852 if (!linkback || curr_lcla == -EINVAL) {
853 unsigned int flags = 0;
e65889c7 854
0c842b55
RV		 855 if (curr_lcla == -EINVAL)
856 flags |= LLI_TERM_INT;
e65889c7 857
0c842b55
RV		 858 d40_log_lli_lcpa_write(chan->lcpa,
859 &lli->dst[lli_current],
860 &lli->src[lli_current],
861 curr_lcla,
862 flags);
863 lli_current++;
864 }
6045f0bb
RV		 865
866 if (curr_lcla < 0)
867 goto out;
868
e65889c7
RV		 869 for (; lli_current < lli_len; lli_current++) {
870 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
871 8 * curr_lcla * 2;
872 struct d40_log_lli *lcla = pool->base + lcla_offset;
0c842b55 873 unsigned int flags = 0;
e65889c7
RV		 874 int next_lcla;
875
876 if (lli_current + 1 < lli_len)
877 next_lcla = d40_lcla_alloc_one(chan, desc);
878 else
0c842b55
RV		 879 next_lcla = linkback ? first_lcla : -EINVAL;
880
881 if (cyclic || next_lcla == -EINVAL)
882 flags |= LLI_TERM_INT;
e65889c7 883
0c842b55
RV		 884 if (linkback && curr_lcla == first_lcla) {
885 /* First link goes in both LCPA and LCLA */
886 d40_log_lli_lcpa_write(chan->lcpa,
887 &lli->dst[lli_current],
888 &lli->src[lli_current],
889 next_lcla, flags);
890 }
891
892 /*
893 * One unused LCLA in the cyclic case if the very first
894 * next_lcla fails...
895 */
e65889c7
RV		 896 d40_log_lli_lcla_write(lcla,
897 &lli->dst[lli_current],
898 &lli->src[lli_current],
0c842b55 899 next_lcla, flags);
e65889c7 900
28c7a19d
N		 901 /*
902 * Cache maintenance is not needed if lcla is
903 * mapped in esram
904 */
905 if (!use_esram_lcla) {
906 dma_sync_single_range_for_device(chan->base->dev,
907 pool->dma_addr, lcla_offset,
908 2 * sizeof(struct d40_log_lli),
909 DMA_TO_DEVICE);
910 }
e65889c7
RV		 911 curr_lcla = next_lcla;
912
0c842b55 913 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
e65889c7
RV		 914 lli_current++;
915 break;
916 }
917 }
918
6045f0bb 919out:
e65889c7
RV		 920 desc->lli_current = lli_current;
921}
698e4732 922
e65889c7
RV		 923static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
924{
724a8577 925 if (chan_is_physical(d40c)) {
1c4b0927 926 d40_phy_lli_load(d40c, d40d);
698e4732 927 d40d->lli_current = d40d->lli_len;
e65889c7
RV		 928 } else
929 d40_log_lli_to_lcxa(d40c, d40d);
698e4732
JA		 930}
931
8d318a50
LW		 932static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
933{
934 struct d40_desc *d;
935
936 if (list_empty(&d40c->active))
937 return NULL;
938
939 d = list_first_entry(&d40c->active,
940 struct d40_desc,
941 node);
942 return d;
943}
944
7404368c 945/* remove desc from current queue and add it to the pending_queue */
8d318a50
LW		 946static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
947{
7404368c
PF		 948 d40_desc_remove(desc);
949 desc->is_in_client_list = false;
a8f3067b
PF		 950 list_add_tail(&desc->node, &d40c->pending_queue);
951}
952
953static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
954{
955 struct d40_desc *d;
956
957 if (list_empty(&d40c->pending_queue))
958 return NULL;
959
960 d = list_first_entry(&d40c->pending_queue,
961 struct d40_desc,
962 node);
963 return d;
8d318a50
LW		 964}
965
966static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
967{
968 struct d40_desc *d;
969
970 if (list_empty(&d40c->queue))
971 return NULL;
972
973 d = list_first_entry(&d40c->queue,
974 struct d40_desc,
975 node);
976 return d;
977}
978
4226dd86
FB		 979static struct d40_desc *d40_first_done(struct d40_chan *d40c)
980{
981 if (list_empty(&d40c->done))
982 return NULL;
983
984 return list_first_entry(&d40c->done, struct d40_desc, node);
985}
986
d49278e3
PF		 987static int d40_psize_2_burst_size(bool is_log, int psize)
988{
989 if (is_log) {
990 if (psize == STEDMA40_PSIZE_LOG_1)
991 return 1;
992 } else {
993 if (psize == STEDMA40_PSIZE_PHY_1)
994 return 1;
995 }
996
997 return 2 << psize;
998}
999
1000/*
1001 * The dma only supports transmitting packages up to
1002 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
1003 * dma elements required to send the entire sg list
1004 */
1005static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
1006{
1007 int dmalen;
1008 u32 max_w = max(data_width1, data_width2);
1009 u32 min_w = min(data_width1, data_width2);
1010 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
1011
1012 if (seg_max > STEDMA40_MAX_SEG_SIZE)
1013 seg_max -= (1 << max_w);
1014
1015 if (!IS_ALIGNED(size, 1 << max_w))
1016 return -EINVAL;
1017
1018 if (size <= seg_max)
1019 dmalen = 1;
1020 else {
1021 dmalen = size / seg_max;
1022 if (dmalen * seg_max < size)
1023 dmalen++;
1024 }
1025 return dmalen;
1026}
1027
1028static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
1029 u32 data_width1, u32 data_width2)
1030{
1031 struct scatterlist *sg;
1032 int i;
1033 int len = 0;
1034 int ret;
1035
1036 for_each_sg(sgl, sg, sg_len, i) {
1037 ret = d40_size_2_dmalen(sg_dma_len(sg),
1038 data_width1, data_width2);
1039 if (ret < 0)
1040 return ret;
1041 len += ret;
1042 }
1043 return len;
1044}
8d318a50 1045
7fb3e75e
N		 1046
1047#ifdef CONFIG_PM
1048static void dma40_backup(void __iomem *baseaddr, u32 *backup,
1049 u32 *regaddr, int num, bool save)
1050{
1051 int i;
1052
1053 for (i = 0; i < num; i++) {
1054 void __iomem *addr = baseaddr + regaddr[i];
1055
1056 if (save)
1057 backup[i] = readl_relaxed(addr);
1058 else
1059 writel_relaxed(backup[i], addr);
1060 }
1061}
1062
1063static void d40_save_restore_registers(struct d40_base *base, bool save)
1064{
1065 int i;
1066
1067 /* Save/Restore channel specific registers */
1068 for (i = 0; i < base->num_phy_chans; i++) {
1069 void __iomem *addr;
1070 int idx;
1071
1072 if (base->phy_res[i].reserved)
1073 continue;
1074
1075 addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
1076 idx = i * ARRAY_SIZE(d40_backup_regs_chan);
1077
1078 dma40_backup(addr, &base->reg_val_backup_chan[idx],
1079 d40_backup_regs_chan,
1080 ARRAY_SIZE(d40_backup_regs_chan),
1081 save);
1082 }
1083
1084 /* Save/Restore global registers */
1085 dma40_backup(base->virtbase, base->reg_val_backup,
1086 d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
1087 save);
1088
1089 /* Save/Restore registers only existing on dma40 v3 and later */
3cb645dc
TL		 1090 if (base->gen_dmac.backup)
1091 dma40_backup(base->virtbase, base->reg_val_backup_v4,
1092 base->gen_dmac.backup,
1093 base->gen_dmac.backup_size,
1094 save);
7fb3e75e
N		 1095}
1096#else
1097static void d40_save_restore_registers(struct d40_base *base, bool save)
1098{
1099}
1100#endif
8d318a50 1101
1bdae6f4
N		 1102static int __d40_execute_command_phy(struct d40_chan *d40c,
1103 enum d40_command command)
8d318a50 1104{
767a9675
JA		 1105 u32 status;
1106 int i;
8d318a50
LW		 1107 void __iomem *active_reg;
1108 int ret = 0;
1109 unsigned long flags;
1d392a7b 1110 u32 wmask;
8d318a50 1111
1bdae6f4
N		 1112 if (command == D40_DMA_STOP) {
1113 ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);
1114 if (ret)
1115 return ret;
1116 }
1117
8d318a50
LW		 1118 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
1119
1120 if (d40c->phy_chan->num % 2 == 0)
1121 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1122 else
1123 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1124
1125 if (command == D40_DMA_SUSPEND_REQ) {
1126 status = (readl(active_reg) &
1127 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1128 D40_CHAN_POS(d40c->phy_chan->num);
1129
1130 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1131 goto done;
1132 }
1133
1d392a7b
JA		 1134 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
1135 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
1136 active_reg);
8d318a50
LW		 1137
1138 if (command == D40_DMA_SUSPEND_REQ) {
1139
1140 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
1141 status = (readl(active_reg) &
1142 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1143 D40_CHAN_POS(d40c->phy_chan->num);
1144
1145 cpu_relax();
1146 /*
1147 * Reduce the number of bus accesses while
1148 * waiting for the DMA to suspend.
1149 */
1150 udelay(3);
1151
1152 if (status == D40_DMA_STOP ||
1153 status == D40_DMA_SUSPENDED)
1154 break;
1155 }
1156
1157 if (i == D40_SUSPEND_MAX_IT) {
6db5a8ba
RV		 1158 chan_err(d40c,
1159 "unable to suspend the chl %d (log: %d) status %x\n",
1160 d40c->phy_chan->num, d40c->log_num,
8d318a50
LW		 1161 status);
1162 dump_stack();
1163 ret = -EBUSY;
1164 }
1165
1166 }
1167done:
1168 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
1169 return ret;
1170}
1171
1172static void d40_term_all(struct d40_chan *d40c)
1173{
1174 struct d40_desc *d40d;
7404368c 1175 struct d40_desc *_d;
8d318a50 1176
4226dd86
FB		 1177 /* Release completed descriptors */
1178 while ((d40d = d40_first_done(d40c))) {
1179 d40_desc_remove(d40d);
1180 d40_desc_free(d40c, d40d);
1181 }
1182
8d318a50
LW		 1183 /* Release active descriptors */
1184 while ((d40d = d40_first_active_get(d40c))) {
1185 d40_desc_remove(d40d);
8d318a50
LW		 1186 d40_desc_free(d40c, d40d);
1187 }
1188
1189 /* Release queued descriptors waiting for transfer */
1190 while ((d40d = d40_first_queued(d40c))) {
1191 d40_desc_remove(d40d);
8d318a50
LW		 1192 d40_desc_free(d40c, d40d);
1193 }
1194
a8f3067b
PF		 1195 /* Release pending descriptors */
1196 while ((d40d = d40_first_pending(d40c))) {
1197 d40_desc_remove(d40d);
1198 d40_desc_free(d40c, d40d);
1199 }
8d318a50 1200
7404368c
PF		 1201 /* Release client owned descriptors */
1202 if (!list_empty(&d40c->client))
1203 list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
1204 d40_desc_remove(d40d);
1205 d40_desc_free(d40c, d40d);
1206 }
1207
82babbb3
PF		 1208 /* Release descriptors in prepare queue */
1209 if (!list_empty(&d40c->prepare_queue))
1210 list_for_each_entry_safe(d40d, _d,
1211 &d40c->prepare_queue, node) {
1212 d40_desc_remove(d40d);
1213 d40_desc_free(d40c, d40d);
1214 }
7404368c 1215
8d318a50 1216 d40c->pending_tx = 0;
8d318a50
LW		 1217}
1218
1bdae6f4
N		 1219static void __d40_config_set_event(struct d40_chan *d40c,
1220 enum d40_events event_type, u32 event,
1221 int reg)
262d2915 1222{
8ca84687 1223 void __iomem *addr = chan_base(d40c) + reg;
262d2915 1224 int tries;
1bdae6f4
N		 1225 u32 status;
1226
1227 switch (event_type) {
1228
1229 case D40_DEACTIVATE_EVENTLINE:
262d2915 1230
262d2915
RV		 1231 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
1232 | ~D40_EVENTLINE_MASK(event), addr);
1bdae6f4
N		 1233 break;
1234
1235 case D40_SUSPEND_REQ_EVENTLINE:
1236 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1237 D40_EVENTLINE_POS(event);
1238
1239 if (status == D40_DEACTIVATE_EVENTLINE ||
1240 status == D40_SUSPEND_REQ_EVENTLINE)
1241 break;
262d2915 1242
1bdae6f4
N		 1243 writel((D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))
1244 | ~D40_EVENTLINE_MASK(event), addr);
1245
1246 for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
1247
1248 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1249 D40_EVENTLINE_POS(event);
1250
1251 cpu_relax();
1252 /*
1253 * Reduce the number of bus accesses while
1254 * waiting for the DMA to suspend.
1255 */
1256 udelay(3);
1257
1258 if (status == D40_DEACTIVATE_EVENTLINE)
1259 break;
1260 }
1261
1262 if (tries == D40_SUSPEND_MAX_IT) {
1263 chan_err(d40c,
1264 "unable to stop the event_line chl %d (log: %d)"
1265 "status %x\n", d40c->phy_chan->num,
1266 d40c->log_num, status);
1267 }
1268 break;
1269
1270 case D40_ACTIVATE_EVENTLINE:
262d2915
RV		 1271 /*
1272 * The hardware sometimes doesn't register the enable when src and dst
1273 * event lines are active on the same logical channel. Retry to ensure
1274 * it does. Usually only one retry is sufficient.
1275 */
1bdae6f4
N		 1276 tries = 100;
1277 while (--tries) {
1278 writel((D40_ACTIVATE_EVENTLINE <<
1279 D40_EVENTLINE_POS(event)) |
1280 ~D40_EVENTLINE_MASK(event), addr);
262d2915 1281
1bdae6f4
N		 1282 if (readl(addr) & D40_EVENTLINE_MASK(event))
1283 break;
1284 }
262d2915 1285
1bdae6f4
N		 1286 if (tries != 99)
1287 dev_dbg(chan2dev(d40c),
1288 "[%s] workaround enable S%cLNK (%d tries)\n",
1289 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
1290 100 - tries);
262d2915 1291
1bdae6f4
N		 1292 WARN_ON(!tries);
1293 break;
262d2915 1294
1bdae6f4
N		 1295 case D40_ROUND_EVENTLINE:
1296 BUG();
1297 break;
8d318a50 1298
1bdae6f4
N		 1299 }
1300}
8d318a50 1301
1bdae6f4
N		 1302static void d40_config_set_event(struct d40_chan *d40c,
1303 enum d40_events event_type)
1304{
26955c07
LJ		 1305 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
1306
8d318a50
LW		 1307 /* Enable event line connected to device (or memcpy) */
1308 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
26955c07 1309 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1bdae6f4 1310 __d40_config_set_event(d40c, event_type, event,
262d2915 1311 D40_CHAN_REG_SSLNK);
8d318a50 1312
26955c07 1313 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM)
1bdae6f4 1314 __d40_config_set_event(d40c, event_type, event,
262d2915 1315 D40_CHAN_REG_SDLNK);
8d318a50
LW		 1316}
1317
a5ebca47 1318static u32 d40_chan_has_events(struct d40_chan *d40c)
8d318a50 1319{
8ca84687 1320 void __iomem *chanbase = chan_base(d40c);
be8cb7df 1321 u32 val;
8d318a50 1322
8ca84687
RV		 1323 val = readl(chanbase + D40_CHAN_REG_SSLNK);
1324 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
be8cb7df 1325
a5ebca47 1326 return val;
8d318a50
LW		 1327}
1328
1bdae6f4
N		 1329static int
1330__d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
1331{
1332 unsigned long flags;
1333 int ret = 0;
1334 u32 active_status;
1335 void __iomem *active_reg;
1336
1337 if (d40c->phy_chan->num % 2 == 0)
1338 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1339 else
1340 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1341
1342
1343 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
1344
1345 switch (command) {
1346 case D40_DMA_STOP:
1347 case D40_DMA_SUSPEND_REQ:
1348
1349 active_status = (readl(active_reg) &
1350 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1351 D40_CHAN_POS(d40c->phy_chan->num);
1352
1353 if (active_status == D40_DMA_RUN)
1354 d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);
1355 else
1356 d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);
1357
1358 if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
1359 ret = __d40_execute_command_phy(d40c, command);
1360
1361 break;
1362
1363 case D40_DMA_RUN:
1364
1365 d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);
1366 ret = __d40_execute_command_phy(d40c, command);
1367 break;
1368
1369 case D40_DMA_SUSPENDED:
1370 BUG();
1371 break;
1372 }
1373
1374 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
1375 return ret;
1376}
1377
1378static int d40_channel_execute_command(struct d40_chan *d40c,
1379 enum d40_command command)
1380{
1381 if (chan_is_logical(d40c))
1382 return __d40_execute_command_log(d40c, command);
1383 else
1384 return __d40_execute_command_phy(d40c, command);
1385}
1386
20a5b6d0
RV		 1387static u32 d40_get_prmo(struct d40_chan *d40c)
1388{
1389 static const unsigned int phy_map[] = {
1390 [STEDMA40_PCHAN_BASIC_MODE]
1391 = D40_DREG_PRMO_PCHAN_BASIC,
1392 [STEDMA40_PCHAN_MODULO_MODE]
1393 = D40_DREG_PRMO_PCHAN_MODULO,
1394 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
1395 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
1396 };
1397 static const unsigned int log_map[] = {
1398 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
1399 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
1400 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
1401 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
1402 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
1403 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
1404 };
1405
724a8577 1406 if (chan_is_physical(d40c))
20a5b6d0
RV		 1407 return phy_map[d40c->dma_cfg.mode_opt];
1408 else
1409 return log_map[d40c->dma_cfg.mode_opt];
1410}
1411
b55912c6 1412static void d40_config_write(struct d40_chan *d40c)
8d318a50
LW		 1413{
1414 u32 addr_base;
1415 u32 var;
8d318a50
LW		 1416
1417 /* Odd addresses are even addresses + 4 */
1418 addr_base = (d40c->phy_chan->num % 2) * 4;
1419 /* Setup channel mode to logical or physical */
724a8577 1420 var = ((u32)(chan_is_logical(d40c)) + 1) <<
8d318a50
LW		 1421 D40_CHAN_POS(d40c->phy_chan->num);
1422 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
1423
1424 /* Setup operational mode option register */
20a5b6d0 1425 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
8d318a50
LW		 1426
1427 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
1428
724a8577 1429 if (chan_is_logical(d40c)) {
8ca84687
RV		 1430 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
1431 & D40_SREG_ELEM_LOG_LIDX_MASK;
1432 void __iomem *chanbase = chan_base(d40c);
1433
8d318a50 1434 /* Set default config for CFG reg */
8ca84687
RV		 1435 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
1436 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
8d318a50 1437
b55912c6 1438 /* Set LIDX for lcla */
8ca84687
RV		 1439 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
1440 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
e9f3a49c
RV		 1441
1442 /* Clear LNK which will be used by d40_chan_has_events() */
1443 writel(0, chanbase + D40_CHAN_REG_SSLNK);
1444 writel(0, chanbase + D40_CHAN_REG_SDLNK);
8d318a50 1445 }
8d318a50
LW		 1446}
1447
aa182ae2
JA		 1448static u32 d40_residue(struct d40_chan *d40c)
1449{
1450 u32 num_elt;
1451
724a8577 1452 if (chan_is_logical(d40c))
aa182ae2
JA		 1453 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1454 >> D40_MEM_LCSP2_ECNT_POS;
8ca84687
RV		 1455 else {
1456 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
1457 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
1458 >> D40_SREG_ELEM_PHY_ECNT_POS;
1459 }
1460
aa182ae2
JA		 1461 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1462}
1463
1464static bool d40_tx_is_linked(struct d40_chan *d40c)
1465{
1466 bool is_link;
1467
724a8577 1468 if (chan_is_logical(d40c))
aa182ae2
JA		 1469 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1470 else
8ca84687
RV		 1471 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
1472 & D40_SREG_LNK_PHYS_LNK_MASK;
1473
aa182ae2
JA		 1474 return is_link;
1475}
1476
86eb5fb6 1477static int d40_pause(struct d40_chan *d40c)
aa182ae2 1478{
aa182ae2
JA		 1479 int res = 0;
1480 unsigned long flags;
1481
3ac012af
JA		 1482 if (!d40c->busy)
1483 return 0;
1484
7fb3e75e 1485 pm_runtime_get_sync(d40c->base->dev);
aa182ae2
JA		 1486 spin_lock_irqsave(&d40c->lock, flags);
1487
1488 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1bdae6f4 1489
7fb3e75e
N		 1490 pm_runtime_mark_last_busy(d40c->base->dev);
1491 pm_runtime_put_autosuspend(d40c->base->dev);
aa182ae2
JA		 1492 spin_unlock_irqrestore(&d40c->lock, flags);
1493 return res;
1494}
1495
86eb5fb6 1496static int d40_resume(struct d40_chan *d40c)
aa182ae2 1497{
aa182ae2
JA		 1498 int res = 0;
1499 unsigned long flags;
1500
3ac012af
JA		 1501 if (!d40c->busy)
1502 return 0;
1503
aa182ae2 1504 spin_lock_irqsave(&d40c->lock, flags);
7fb3e75e 1505 pm_runtime_get_sync(d40c->base->dev);
aa182ae2
JA		 1506
1507 /* If bytes left to transfer or linked tx resume job */
1bdae6f4 1508 if (d40_residue(d40c) || d40_tx_is_linked(d40c))
aa182ae2 1509 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
aa182ae2 1510
7fb3e75e
N		 1511 pm_runtime_mark_last_busy(d40c->base->dev);
1512 pm_runtime_put_autosuspend(d40c->base->dev);
aa182ae2
JA		 1513 spin_unlock_irqrestore(&d40c->lock, flags);
1514 return res;
1515}
1516
8d318a50
LW		 1517static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1518{
1519 struct d40_chan *d40c = container_of(tx->chan,
1520 struct d40_chan,
1521 chan);
1522 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1523 unsigned long flags;
884485e1 1524 dma_cookie_t cookie;
8d318a50
LW		 1525
1526 spin_lock_irqsave(&d40c->lock, flags);
884485e1 1527 cookie = dma_cookie_assign(tx);
8d318a50 1528 d40_desc_queue(d40c, d40d);
8d318a50
LW		 1529 spin_unlock_irqrestore(&d40c->lock, flags);
1530
884485e1 1531 return cookie;
8d318a50
LW		 1532}
1533
1534static int d40_start(struct d40_chan *d40c)
1535{
0c32269d 1536 return d40_channel_execute_command(d40c, D40_DMA_RUN);
8d318a50
LW		 1537}
1538
1539static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1540{
1541 struct d40_desc *d40d;
1542 int err;
1543
1544 /* Start queued jobs, if any */
1545 d40d = d40_first_queued(d40c);
1546
1547 if (d40d != NULL) {
1bdae6f4 1548 if (!d40c->busy) {
7fb3e75e 1549 d40c->busy = true;
1bdae6f4
N		 1550 pm_runtime_get_sync(d40c->base->dev);
1551 }
8d318a50
LW		 1552
1553 /* Remove from queue */
1554 d40_desc_remove(d40d);
1555
1556 /* Add to active queue */
1557 d40_desc_submit(d40c, d40d);
1558
7d83a854
RV		 1559 /* Initiate DMA job */
1560 d40_desc_load(d40c, d40d);
8d318a50 1561
7d83a854
RV		 1562 /* Start dma job */
1563 err = d40_start(d40c);
8d318a50 1564
7d83a854
RV		 1565 if (err)
1566 return NULL;
8d318a50
LW		 1567 }
1568
1569 return d40d;
1570}
1571
1572/* called from interrupt context */
1573static void dma_tc_handle(struct d40_chan *d40c)
1574{
1575 struct d40_desc *d40d;
1576
8d318a50
LW		 1577 /* Get first active entry from list */
1578 d40d = d40_first_active_get(d40c);
1579
1580 if (d40d == NULL)
1581 return;
1582
0c842b55
RV		 1583 if (d40d->cyclic) {
1584 /*
1585 * If this was a paritially loaded list, we need to reloaded
1586 * it, and only when the list is completed. We need to check
1587 * for done because the interrupt will hit for every link, and
1588 * not just the last one.
1589 */
1590 if (d40d->lli_current < d40d->lli_len
1591 && !d40_tx_is_linked(d40c)
1592 && !d40_residue(d40c)) {
1593 d40_lcla_free_all(d40c, d40d);
1594 d40_desc_load(d40c, d40d);
1595 (void) d40_start(d40c);
8d318a50 1596
0c842b55
RV		 1597 if (d40d->lli_current == d40d->lli_len)
1598 d40d->lli_current = 0;
1599 }
1600 } else {
1601 d40_lcla_free_all(d40c, d40d);
8d318a50 1602
0c842b55
RV		 1603 if (d40d->lli_current < d40d->lli_len) {
1604 d40_desc_load(d40c, d40d);
1605 /* Start dma job */
1606 (void) d40_start(d40c);
1607 return;
1608 }
1609
1610 if (d40_queue_start(d40c) == NULL)
1611 d40c->busy = false;
7fb3e75e
N		 1612 pm_runtime_mark_last_busy(d40c->base->dev);
1613 pm_runtime_put_autosuspend(d40c->base->dev);
8d318a50 1614
7dd14525
FB		 1615 d40_desc_remove(d40d);
1616 d40_desc_done(d40c, d40d);
1617 }
4226dd86 1618
8d318a50
LW		 1619 d40c->pending_tx++;
1620 tasklet_schedule(&d40c->tasklet);
1621
1622}
1623
1624static void dma_tasklet(unsigned long data)
1625{
1626 struct d40_chan *d40c = (struct d40_chan *) data;
767a9675 1627 struct d40_desc *d40d;
8d318a50
LW		 1628 unsigned long flags;
1629 dma_async_tx_callback callback;
1630 void *callback_param;
1631
1632 spin_lock_irqsave(&d40c->lock, flags);
1633
4226dd86
FB		 1634 /* Get first entry from the done list */
1635 d40d = d40_first_done(d40c);
1636 if (d40d == NULL) {
1637 /* Check if we have reached here for cyclic job */
1638 d40d = d40_first_active_get(d40c);
1639 if (d40d == NULL || !d40d->cyclic)
1640 goto err;
1641 }
8d318a50 1642
0c842b55 1643 if (!d40d->cyclic)
f7fbce07 1644 dma_cookie_complete(&d40d->txd);
8d318a50
LW		 1645
1646 /*
1647 * If terminating a channel pending_tx is set to zero.
1648 * This prevents any finished active jobs to return to the client.
1649 */
1650 if (d40c->pending_tx == 0) {
1651 spin_unlock_irqrestore(&d40c->lock, flags);
1652 return;
1653 }
1654
1655 /* Callback to client */
767a9675
JA		 1656 callback = d40d->txd.callback;
1657 callback_param = d40d->txd.callback_param;
1658
0c842b55
RV		 1659 if (!d40d->cyclic) {
1660 if (async_tx_test_ack(&d40d->txd)) {
767a9675 1661 d40_desc_remove(d40d);
0c842b55 1662 d40_desc_free(d40c, d40d);
f26e03ad
FB		 1663 } else if (!d40d->is_in_client_list) {
1664 d40_desc_remove(d40d);
1665 d40_lcla_free_all(d40c, d40d);
1666 list_add_tail(&d40d->node, &d40c->client);
1667 d40d->is_in_client_list = true;
8d318a50
LW		 1668 }
1669 }
1670
1671 d40c->pending_tx--;
1672
1673 if (d40c->pending_tx)
1674 tasklet_schedule(&d40c->tasklet);
1675
1676 spin_unlock_irqrestore(&d40c->lock, flags);
1677
767a9675 1678 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
8d318a50
LW		 1679 callback(callback_param);
1680
1681 return;
1682
1bdae6f4
N		 1683err:
1684 /* Rescue manouver if receiving double interrupts */
8d318a50
LW		 1685 if (d40c->pending_tx > 0)
1686 d40c->pending_tx--;
1687 spin_unlock_irqrestore(&d40c->lock, flags);
1688}
1689
1690static irqreturn_t d40_handle_interrupt(int irq, void *data)
1691{
8d318a50 1692 int i;
8d318a50
LW		 1693 u32 idx;
1694 u32 row;
1695 long chan = -1;
1696 struct d40_chan *d40c;
1697 unsigned long flags;
1698 struct d40_base *base = data;
3cb645dc
TL		 1699 u32 regs[base->gen_dmac.il_size];
1700 struct d40_interrupt_lookup *il = base->gen_dmac.il;
1701 u32 il_size = base->gen_dmac.il_size;
8d318a50
LW		 1702
1703 spin_lock_irqsave(&base->interrupt_lock, flags);
1704
1705 /* Read interrupt status of both logical and physical channels */
3cb645dc 1706 for (i = 0; i < il_size; i++)
8d318a50
LW		 1707 regs[i] = readl(base->virtbase + il[i].src);
1708
1709 for (;;) {
1710
1711 chan = find_next_bit((unsigned long *)regs,
3cb645dc 1712 BITS_PER_LONG * il_size, chan + 1);
8d318a50
LW		 1713
1714 /* No more set bits found? */
3cb645dc 1715 if (chan == BITS_PER_LONG * il_size)
8d318a50
LW		 1716 break;
1717
1718 row = chan / BITS_PER_LONG;
1719 idx = chan & (BITS_PER_LONG - 1);
1720
8d318a50
LW		 1721 if (il[row].offset == D40_PHY_CHAN)
1722 d40c = base->lookup_phy_chans[idx];
1723 else
1724 d40c = base->lookup_log_chans[il[row].offset + idx];
53d6d68f
FB		 1725
1726 if (!d40c) {
1727 /*
1728 * No error because this can happen if something else
1729 * in the system is using the channel.
1730 */
1731 continue;
1732 }
1733
1734 /* ACK interrupt */
1735 writel(1 << idx, base->virtbase + il[row].clr);
1736
8d318a50
LW		 1737 spin_lock(&d40c->lock);
1738
1739 if (!il[row].is_error)
1740 dma_tc_handle(d40c);
1741 else
6db5a8ba
RV		 1742 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1743 chan, il[row].offset, idx);
8d318a50
LW		 1744
1745 spin_unlock(&d40c->lock);
1746 }
1747
1748 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1749
1750 return IRQ_HANDLED;
1751}
1752
8d318a50
LW		 1753static int d40_validate_conf(struct d40_chan *d40c,
1754 struct stedma40_chan_cfg *conf)
1755{
1756 int res = 0;
38bdbf02 1757 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
8d318a50 1758
0747c7ba 1759 if (!conf->dir) {
6db5a8ba 1760 chan_err(d40c, "Invalid direction.\n");
0747c7ba
LW		 1761 res = -EINVAL;
1762 }
1763
26955c07
LJ		 1764 if ((is_log && conf->dev_type > d40c->base->num_log_chans) ||
1765 (!is_log && conf->dev_type > d40c->base->num_phy_chans) ||
1766 (conf->dev_type < 0)) {
1767 chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type);
0747c7ba
LW		 1768 res = -EINVAL;
1769 }
1770
1771 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
26955c07
LJ		 1772 d40c->base->plat_data->dev_tx[conf->dev_type] == 0 &&
1773 d40c->runtime_addr == 0) {
1774 chan_err(d40c, "Invalid TX channel address (%d)\n",
1775 conf->dev_type);
8d318a50
LW		 1776 res = -EINVAL;
1777 }
1778
0747c7ba 1779 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
26955c07
LJ		 1780 d40c->base->plat_data->dev_rx[conf->dev_type] == 0 &&
1781 d40c->runtime_addr == 0) {
1782 chan_err(d40c, "Invalid RX channel address (%d)\n",
1783 conf->dev_type);
8d318a50
LW		 1784 res = -EINVAL;
1785 }
1786
1787 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1788 /*
1789 * DMAC HW supports it. Will be added to this driver,
1790 * in case any dma client requires it.
1791 */
6db5a8ba 1792 chan_err(d40c, "periph to periph not supported\n");
8d318a50
LW		 1793 res = -EINVAL;
1794 }
1795
d49278e3
PF		 1796 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1797 (1 << conf->src_info.data_width) !=
1798 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1799 (1 << conf->dst_info.data_width)) {
1800 /*
1801 * The DMAC hardware only supports
1802 * src (burst x width) == dst (burst x width)
1803 */
1804
6db5a8ba 1805 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
d49278e3
PF		 1806 res = -EINVAL;
1807 }
1808
8d318a50
LW		 1809 return res;
1810}
1811
5cd326fd
N		 1812static bool d40_alloc_mask_set(struct d40_phy_res *phy,
1813 bool is_src, int log_event_line, bool is_log,
1814 bool *first_user)
8d318a50
LW		 1815{
1816 unsigned long flags;
1817 spin_lock_irqsave(&phy->lock, flags);
5cd326fd
N		 1818
1819 *first_user = ((phy->allocated_src | phy->allocated_dst)
1820 == D40_ALLOC_FREE);
1821
4aed79b2 1822 if (!is_log) {
8d318a50
LW		 1823 /* Physical interrupts are masked per physical full channel */
1824 if (phy->allocated_src == D40_ALLOC_FREE &&
1825 phy->allocated_dst == D40_ALLOC_FREE) {
1826 phy->allocated_dst = D40_ALLOC_PHY;
1827 phy->allocated_src = D40_ALLOC_PHY;
1828 goto found;
1829 } else
1830 goto not_found;
1831 }
1832
1833 /* Logical channel */
1834 if (is_src) {
1835 if (phy->allocated_src == D40_ALLOC_PHY)
1836 goto not_found;
1837
1838 if (phy->allocated_src == D40_ALLOC_FREE)
1839 phy->allocated_src = D40_ALLOC_LOG_FREE;
1840
1841 if (!(phy->allocated_src & (1 << log_event_line))) {
1842 phy->allocated_src |= 1 << log_event_line;
1843 goto found;
1844 } else
1845 goto not_found;
1846 } else {
1847 if (phy->allocated_dst == D40_ALLOC_PHY)
1848 goto not_found;
1849
1850 if (phy->allocated_dst == D40_ALLOC_FREE)
1851 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1852
1853 if (!(phy->allocated_dst & (1 << log_event_line))) {
1854 phy->allocated_dst |= 1 << log_event_line;
1855 goto found;
1856 } else
1857 goto not_found;
1858 }
1859
1860not_found:
1861 spin_unlock_irqrestore(&phy->lock, flags);
1862 return false;
1863found:
1864 spin_unlock_irqrestore(&phy->lock, flags);
1865 return true;
1866}
1867
1868static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1869 int log_event_line)
1870{
1871 unsigned long flags;
1872 bool is_free = false;
1873
1874 spin_lock_irqsave(&phy->lock, flags);
1875 if (!log_event_line) {
8d318a50
LW		 1876 phy->allocated_dst = D40_ALLOC_FREE;
1877 phy->allocated_src = D40_ALLOC_FREE;
1878 is_free = true;
1879 goto out;
1880 }
1881
1882 /* Logical channel */
1883 if (is_src) {
1884 phy->allocated_src &= ~(1 << log_event_line);
1885 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1886 phy->allocated_src = D40_ALLOC_FREE;
1887 } else {
1888 phy->allocated_dst &= ~(1 << log_event_line);
1889 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1890 phy->allocated_dst = D40_ALLOC_FREE;
1891 }
1892
1893 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1894 D40_ALLOC_FREE);
1895
1896out:
1897 spin_unlock_irqrestore(&phy->lock, flags);
1898
1899 return is_free;
1900}
1901
5cd326fd 1902static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
8d318a50 1903{
26955c07 1904 int dev_type = d40c->dma_cfg.dev_type;
8d318a50
LW		 1905 int event_group;
1906 int event_line;
1907 struct d40_phy_res *phys;
1908 int i;
1909 int j;
1910 int log_num;
f000df8c 1911 int num_phy_chans;
8d318a50 1912 bool is_src;
38bdbf02 1913 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
8d318a50
LW		 1914
1915 phys = d40c->base->phy_res;
f000df8c 1916 num_phy_chans = d40c->base->num_phy_chans;
8d318a50
LW		 1917
1918 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
8d318a50
LW		 1919 log_num = 2 * dev_type;
1920 is_src = true;
1921 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1922 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1923 /* dst event lines are used for logical memcpy */
8d318a50
LW		 1924 log_num = 2 * dev_type + 1;
1925 is_src = false;
1926 } else
1927 return -EINVAL;
1928
1929 event_group = D40_TYPE_TO_GROUP(dev_type);
1930 event_line = D40_TYPE_TO_EVENT(dev_type);
1931
1932 if (!is_log) {
1933 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1934 /* Find physical half channel */
f000df8c
GB		 1935 if (d40c->dma_cfg.use_fixed_channel) {
1936 i = d40c->dma_cfg.phy_channel;
4aed79b2 1937 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 1938 0, is_log,
1939 first_phy_user))
8d318a50 1940 goto found_phy;
f000df8c
GB		 1941 } else {
1942 for (i = 0; i < num_phy_chans; i++) {
1943 if (d40_alloc_mask_set(&phys[i], is_src,
1944 0, is_log,
1945 first_phy_user))
1946 goto found_phy;
1947 }
8d318a50
LW		 1948 }
1949 } else
1950 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1951 int phy_num = j + event_group * 2;
1952 for (i = phy_num; i < phy_num + 2; i++) {
508849ad
LW		 1953 if (d40_alloc_mask_set(&phys[i],
1954 is_src,
1955 0,
5cd326fd
N		 1956 is_log,
1957 first_phy_user))
8d318a50
LW		 1958 goto found_phy;
1959 }
1960 }
1961 return -EINVAL;
1962found_phy:
1963 d40c->phy_chan = &phys[i];
1964 d40c->log_num = D40_PHY_CHAN;
1965 goto out;
1966 }
1967 if (dev_type == -1)
1968 return -EINVAL;
1969
1970 /* Find logical channel */
1971 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1972 int phy_num = j + event_group * 2;
5cd326fd
N		 1973
1974 if (d40c->dma_cfg.use_fixed_channel) {
1975 i = d40c->dma_cfg.phy_channel;
1976
1977 if ((i != phy_num) && (i != phy_num + 1)) {
1978 dev_err(chan2dev(d40c),
1979 "invalid fixed phy channel %d\n", i);
1980 return -EINVAL;
1981 }
1982
1983 if (d40_alloc_mask_set(&phys[i], is_src, event_line,
1984 is_log, first_phy_user))
1985 goto found_log;
1986
1987 dev_err(chan2dev(d40c),
1988 "could not allocate fixed phy channel %d\n", i);
1989 return -EINVAL;
1990 }
1991
8d318a50
LW		 1992 /*
1993 * Spread logical channels across all available physical rather
1994 * than pack every logical channel at the first available phy
1995 * channels.
1996 */
1997 if (is_src) {
1998 for (i = phy_num; i < phy_num + 2; i++) {
1999 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 2000 event_line, is_log,
2001 first_phy_user))
8d318a50
LW		 2002 goto found_log;
2003 }
2004 } else {
2005 for (i = phy_num + 1; i >= phy_num; i--) {
2006 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 2007 event_line, is_log,
2008 first_phy_user))
8d318a50
LW		 2009 goto found_log;
2010 }
2011 }
2012 }
2013 return -EINVAL;
2014
2015found_log:
2016 d40c->phy_chan = &phys[i];
2017 d40c->log_num = log_num;
2018out:
2019
2020 if (is_log)
2021 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
2022 else
2023 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
2024
2025 return 0;
2026
2027}
2028
8d318a50
LW		 2029static int d40_config_memcpy(struct d40_chan *d40c)
2030{
2031 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
2032
2033 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
29027a1e 2034 d40c->dma_cfg = dma40_memcpy_conf_log;
26955c07 2035 d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id];
8d318a50
LW		 2036
2037 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
2038 dma_has_cap(DMA_SLAVE, cap)) {
29027a1e 2039 d40c->dma_cfg = dma40_memcpy_conf_phy;
8d318a50 2040 } else {
6db5a8ba 2041 chan_err(d40c, "No memcpy\n");
8d318a50
LW		 2042 return -EINVAL;
2043 }
2044
2045 return 0;
2046}
2047
8d318a50
LW		 2048static int d40_free_dma(struct d40_chan *d40c)
2049{
2050
2051 int res = 0;
26955c07 2052 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
8d318a50
LW		 2053 struct d40_phy_res *phy = d40c->phy_chan;
2054 bool is_src;
2055
2056 /* Terminate all queued and active transfers */
2057 d40_term_all(d40c);
2058
2059 if (phy == NULL) {
6db5a8ba 2060 chan_err(d40c, "phy == null\n");
8d318a50
LW		 2061 return -EINVAL;
2062 }
2063
2064 if (phy->allocated_src == D40_ALLOC_FREE &&
2065 phy->allocated_dst == D40_ALLOC_FREE) {
6db5a8ba 2066 chan_err(d40c, "channel already free\n");
8d318a50
LW		 2067 return -EINVAL;
2068 }
2069
8d318a50 2070 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
26955c07 2071 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
8d318a50 2072 is_src = false;
26955c07 2073 else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
8d318a50 2074 is_src = true;
26955c07 2075 else {
6db5a8ba 2076 chan_err(d40c, "Unknown direction\n");
8d318a50
LW		 2077 return -EINVAL;
2078 }
2079
7fb3e75e 2080 pm_runtime_get_sync(d40c->base->dev);
1bdae6f4 2081 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
d181b3a8 2082 if (res) {
1bdae6f4 2083 chan_err(d40c, "stop failed\n");
7fb3e75e 2084 goto out;
d181b3a8
JA		 2085 }
2086
1bdae6f4 2087 d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
8d318a50 2088
1bdae6f4 2089 if (chan_is_logical(d40c))
8d318a50 2090 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1bdae6f4
N		 2091 else
2092 d40c->base->lookup_phy_chans[phy->num] = NULL;
7fb3e75e
N		 2093
2094 if (d40c->busy) {
2095 pm_runtime_mark_last_busy(d40c->base->dev);
2096 pm_runtime_put_autosuspend(d40c->base->dev);
2097 }
2098
2099 d40c->busy = false;
8d318a50 2100 d40c->phy_chan = NULL;
ce2ca125 2101 d40c->configured = false;
7fb3e75e 2102out:
8d318a50 2103
7fb3e75e
N		 2104 pm_runtime_mark_last_busy(d40c->base->dev);
2105 pm_runtime_put_autosuspend(d40c->base->dev);
2106 return res;
8d318a50
LW		 2107}
2108
a5ebca47
JA		 2109static bool d40_is_paused(struct d40_chan *d40c)
2110{
8ca84687 2111 void __iomem *chanbase = chan_base(d40c);
a5ebca47
JA		 2112 bool is_paused = false;
2113 unsigned long flags;
2114 void __iomem *active_reg;
2115 u32 status;
26955c07 2116 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
a5ebca47
JA		 2117
2118 spin_lock_irqsave(&d40c->lock, flags);
2119
724a8577 2120 if (chan_is_physical(d40c)) {
a5ebca47
JA		 2121 if (d40c->phy_chan->num % 2 == 0)
2122 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
2123 else
2124 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
2125
2126 status = (readl(active_reg) &
2127 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
2128 D40_CHAN_POS(d40c->phy_chan->num);
2129 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
2130 is_paused = true;
2131
2132 goto _exit;
2133 }
2134
a5ebca47 2135 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
9dbfbd35 2136 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
8ca84687 2137 status = readl(chanbase + D40_CHAN_REG_SDLNK);
9dbfbd35 2138 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
8ca84687 2139 status = readl(chanbase + D40_CHAN_REG_SSLNK);
9dbfbd35 2140 } else {
6db5a8ba 2141 chan_err(d40c, "Unknown direction\n");
a5ebca47
JA		 2142 goto _exit;
2143 }
9dbfbd35 2144
a5ebca47
JA		 2145 status = (status & D40_EVENTLINE_MASK(event)) >>
2146 D40_EVENTLINE_POS(event);
2147
2148 if (status != D40_DMA_RUN)
2149 is_paused = true;
a5ebca47
JA		 2150_exit:
2151 spin_unlock_irqrestore(&d40c->lock, flags);
2152 return is_paused;
2153
2154}
2155
8d318a50
LW		 2156static u32 stedma40_residue(struct dma_chan *chan)
2157{
2158 struct d40_chan *d40c =
2159 container_of(chan, struct d40_chan, chan);
2160 u32 bytes_left;
2161 unsigned long flags;
2162
2163 spin_lock_irqsave(&d40c->lock, flags);
2164 bytes_left = d40_residue(d40c);
2165 spin_unlock_irqrestore(&d40c->lock, flags);
2166
2167 return bytes_left;
2168}
2169
3e3a0763
RV		 2170static int
2171d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
2172 struct scatterlist *sg_src, struct scatterlist *sg_dst,
822c5676
RV		 2173 unsigned int sg_len, dma_addr_t src_dev_addr,
2174 dma_addr_t dst_dev_addr)
3e3a0763
RV		 2175{
2176 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2177 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2178 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
5ed04b85 2179 int ret;
3e3a0763 2180
5ed04b85
RV		 2181 ret = d40_log_sg_to_lli(sg_src, sg_len,
2182 src_dev_addr,
2183 desc->lli_log.src,
2184 chan->log_def.lcsp1,
2185 src_info->data_width,
2186 dst_info->data_width);
2187
2188 ret = d40_log_sg_to_lli(sg_dst, sg_len,
2189 dst_dev_addr,
2190 desc->lli_log.dst,
2191 chan->log_def.lcsp3,
2192 dst_info->data_width,
2193 src_info->data_width);
2194
2195 return ret < 0 ? ret : 0;
3e3a0763
RV		 2196}
2197
2198static int
2199d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
2200 struct scatterlist *sg_src, struct scatterlist *sg_dst,
822c5676
RV		 2201 unsigned int sg_len, dma_addr_t src_dev_addr,
2202 dma_addr_t dst_dev_addr)
3e3a0763 2203{
3e3a0763
RV		 2204 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2205 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2206 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
0c842b55 2207 unsigned long flags = 0;
3e3a0763
RV		 2208 int ret;
2209
0c842b55
RV		 2210 if (desc->cyclic)
2211 flags |= LLI_CYCLIC | LLI_TERM_INT;
2212
3e3a0763
RV		 2213 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
2214 desc->lli_phy.src,
2215 virt_to_phys(desc->lli_phy.src),
2216 chan->src_def_cfg,
0c842b55 2217 src_info, dst_info, flags);
3e3a0763
RV		 2218
2219 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
2220 desc->lli_phy.dst,
2221 virt_to_phys(desc->lli_phy.dst),
2222 chan->dst_def_cfg,
0c842b55 2223 dst_info, src_info, flags);
3e3a0763
RV		 2224
2225 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
2226 desc->lli_pool.size, DMA_TO_DEVICE);
2227
2228 return ret < 0 ? ret : 0;
2229}
2230
5f81158f
RV		 2231static struct d40_desc *
2232d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
2233 unsigned int sg_len, unsigned long dma_flags)
2234{
2235 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2236 struct d40_desc *desc;
dbd88788 2237 int ret;
5f81158f
RV		 2238
2239 desc = d40_desc_get(chan);
2240 if (!desc)
2241 return NULL;
2242
2243 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
2244 cfg->dst_info.data_width);
2245 if (desc->lli_len < 0) {
2246 chan_err(chan, "Unaligned size\n");
dbd88788
RV		 2247 goto err;
2248 }
5f81158f 2249
dbd88788
RV		 2250 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
2251 if (ret < 0) {
2252 chan_err(chan, "Could not allocate lli\n");
2253 goto err;
5f81158f
RV		 2254 }
2255
2256 desc->lli_current = 0;
2257 desc->txd.flags = dma_flags;
2258 desc->txd.tx_submit = d40_tx_submit;
2259
2260 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
2261
2262 return desc;
dbd88788
RV		 2263
2264err:
2265 d40_desc_free(chan, desc);
2266 return NULL;
5f81158f
RV		 2267}
2268
cade1d30 2269static dma_addr_t
db8196df 2270d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
8d318a50 2271{
cade1d30
RV		 2272 struct stedma40_platform_data *plat = chan->base->plat_data;
2273 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
711b9cea 2274 dma_addr_t addr = 0;
cade1d30
RV		 2275
2276 if (chan->runtime_addr)
2277 return chan->runtime_addr;
2278
db8196df 2279 if (direction == DMA_DEV_TO_MEM)
26955c07 2280 addr = plat->dev_rx[cfg->dev_type];
db8196df 2281 else if (direction == DMA_MEM_TO_DEV)
26955c07 2282 addr = plat->dev_tx[cfg->dev_type];
cade1d30
RV		 2283
2284 return addr;
2285}
2286
2287static struct dma_async_tx_descriptor *
2288d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
2289 struct scatterlist *sg_dst, unsigned int sg_len,
db8196df 2290 enum dma_transfer_direction direction, unsigned long dma_flags)
cade1d30
RV		 2291{
2292 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
822c5676
RV		 2293 dma_addr_t src_dev_addr = 0;
2294 dma_addr_t dst_dev_addr = 0;
cade1d30 2295 struct d40_desc *desc;
2a614340 2296 unsigned long flags;
cade1d30 2297 int ret;
8d318a50 2298
cade1d30
RV		 2299 if (!chan->phy_chan) {
2300 chan_err(chan, "Cannot prepare unallocated channel\n");
2301 return NULL;
0d0f6b8b
JA		 2302 }
2303
cade1d30 2304 spin_lock_irqsave(&chan->lock, flags);
8d318a50 2305
cade1d30
RV		 2306 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
2307 if (desc == NULL)
8d318a50
LW		 2308 goto err;
2309
0c842b55
RV		 2310 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
2311 desc->cyclic = true;
2312
7e426da8 2313 if (direction != DMA_TRANS_NONE) {
822c5676
RV		 2314 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
2315
db8196df 2316 if (direction == DMA_DEV_TO_MEM)
822c5676 2317 src_dev_addr = dev_addr;
db8196df 2318 else if (direction == DMA_MEM_TO_DEV)
822c5676
RV		 2319 dst_dev_addr = dev_addr;
2320 }
cade1d30
RV		 2321
2322 if (chan_is_logical(chan))
2323 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
822c5676 2324 sg_len, src_dev_addr, dst_dev_addr);
cade1d30
RV		 2325 else
2326 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
822c5676 2327 sg_len, src_dev_addr, dst_dev_addr);
cade1d30
RV		 2328
2329 if (ret) {
2330 chan_err(chan, "Failed to prepare %s sg job: %d\n",
2331 chan_is_logical(chan) ? "log" : "phy", ret);
2332 goto err;
8d318a50
LW		 2333 }
2334
82babbb3
PF		 2335 /*
2336 * add descriptor to the prepare queue in order to be able
2337 * to free them later in terminate_all
2338 */
2339 list_add_tail(&desc->node, &chan->prepare_queue);
2340
cade1d30
RV		 2341 spin_unlock_irqrestore(&chan->lock, flags);
2342
2343 return &desc->txd;
8d318a50 2344
8d318a50 2345err:
cade1d30
RV		 2346 if (desc)
2347 d40_desc_free(chan, desc);
2348 spin_unlock_irqrestore(&chan->lock, flags);
8d318a50
LW		 2349 return NULL;
2350}
8d318a50
LW		 2351
2352bool stedma40_filter(struct dma_chan *chan, void *data)
2353{
2354 struct stedma40_chan_cfg *info = data;
2355 struct d40_chan *d40c =
2356 container_of(chan, struct d40_chan, chan);
2357 int err;
2358
2359 if (data) {
2360 err = d40_validate_conf(d40c, info);
2361 if (!err)
2362 d40c->dma_cfg = *info;
2363 } else
2364 err = d40_config_memcpy(d40c);
2365
ce2ca125
RV		 2366 if (!err)
2367 d40c->configured = true;
2368
8d318a50
LW		 2369 return err == 0;
2370}
2371EXPORT_SYMBOL(stedma40_filter);
2372
ac2c0a38
RV		 2373static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
2374{
2375 bool realtime = d40c->dma_cfg.realtime;
2376 bool highprio = d40c->dma_cfg.high_priority;
3cb645dc 2377 u32 rtreg;
ac2c0a38
RV		 2378 u32 event = D40_TYPE_TO_EVENT(dev_type);
2379 u32 group = D40_TYPE_TO_GROUP(dev_type);
2380 u32 bit = 1 << event;
ccc3d697 2381 u32 prioreg;
3cb645dc 2382 struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;
ccc3d697 2383
3cb645dc 2384 rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;
ccc3d697
RV		 2385 /*
2386 * Due to a hardware bug, in some cases a logical channel triggered by
2387 * a high priority destination event line can generate extra packet
2388 * transactions.
2389 *
2390 * The workaround is to not set the high priority level for the
2391 * destination event lines that trigger logical channels.
2392 */
2393 if (!src && chan_is_logical(d40c))
2394 highprio = false;
2395
3cb645dc 2396 prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;
ac2c0a38
RV		 2397
2398 /* Destination event lines are stored in the upper halfword */
2399 if (!src)
2400 bit <<= 16;
2401
2402 writel(bit, d40c->base->virtbase + prioreg + group * 4);
2403 writel(bit, d40c->base->virtbase + rtreg + group * 4);
2404}
2405
2406static void d40_set_prio_realtime(struct d40_chan *d40c)
2407{
2408 if (d40c->base->rev < 3)
2409 return;
2410
2411 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
2412 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
26955c07 2413 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, true);
ac2c0a38
RV		 2414
2415 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
2416 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
26955c07 2417 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, false);
ac2c0a38
RV		 2418}
2419
8d318a50
LW		 2420/* DMA ENGINE functions */
2421static int d40_alloc_chan_resources(struct dma_chan *chan)
2422{
2423 int err;
2424 unsigned long flags;
2425 struct d40_chan *d40c =
2426 container_of(chan, struct d40_chan, chan);
ef1872ec 2427 bool is_free_phy;
8d318a50
LW		 2428 spin_lock_irqsave(&d40c->lock, flags);
2429
d3ee98cd 2430 dma_cookie_init(chan);
8d318a50 2431
ce2ca125
RV		 2432 /* If no dma configuration is set use default configuration (memcpy) */
2433 if (!d40c->configured) {
8d318a50 2434 err = d40_config_memcpy(d40c);
ff0b12ba 2435 if (err) {
6db5a8ba 2436 chan_err(d40c, "Failed to configure memcpy channel\n");
ff0b12ba
JA		 2437 goto fail;
2438 }
8d318a50
LW		 2439 }
2440
5cd326fd 2441 err = d40_allocate_channel(d40c, &is_free_phy);
8d318a50 2442 if (err) {
6db5a8ba 2443 chan_err(d40c, "Failed to allocate channel\n");
7fb3e75e 2444 d40c->configured = false;
ff0b12ba 2445 goto fail;
8d318a50
LW		 2446 }
2447
7fb3e75e 2448 pm_runtime_get_sync(d40c->base->dev);
ef1872ec
LW		 2449 /* Fill in basic CFG register values */
2450 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
724a8577 2451 &d40c->dst_def_cfg, chan_is_logical(d40c));
ef1872ec 2452
ac2c0a38
RV		 2453 d40_set_prio_realtime(d40c);
2454
724a8577 2455 if (chan_is_logical(d40c)) {
ef1872ec
LW		 2456 d40_log_cfg(&d40c->dma_cfg,
2457 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2458
2459 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
2460 d40c->lcpa = d40c->base->lcpa_base +
26955c07 2461 d40c->dma_cfg.dev_type * D40_LCPA_CHAN_SIZE;
ef1872ec
LW		 2462 else
2463 d40c->lcpa = d40c->base->lcpa_base +
26955c07 2464 d40c->dma_cfg.dev_type *
f26e03ad 2465 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
ef1872ec
LW		 2466 }
2467
5cd326fd
N		 2468 dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
2469 chan_is_logical(d40c) ? "logical" : "physical",
2470 d40c->phy_chan->num,
2471 d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
2472
2473
ef1872ec
LW		 2474 /*
2475 * Only write channel configuration to the DMA if the physical
2476 * resource is free. In case of multiple logical channels
2477 * on the same physical resource, only the first write is necessary.
2478 */
b55912c6
JA		 2479 if (is_free_phy)
2480 d40_config_write(d40c);
ff0b12ba 2481fail:
7fb3e75e
N		 2482 pm_runtime_mark_last_busy(d40c->base->dev);
2483 pm_runtime_put_autosuspend(d40c->base->dev);
8d318a50 2484 spin_unlock_irqrestore(&d40c->lock, flags);
ff0b12ba 2485 return err;
8d318a50
LW		 2486}
2487
2488static void d40_free_chan_resources(struct dma_chan *chan)
2489{
2490 struct d40_chan *d40c =
2491 container_of(chan, struct d40_chan, chan);
2492 int err;
2493 unsigned long flags;
2494
0d0f6b8b 2495 if (d40c->phy_chan == NULL) {
6db5a8ba 2496 chan_err(d40c, "Cannot free unallocated channel\n");
0d0f6b8b
JA		 2497 return;
2498 }
2499
8d318a50
LW		 2500 spin_lock_irqsave(&d40c->lock, flags);
2501
2502 err = d40_free_dma(d40c);
2503
2504 if (err)
6db5a8ba 2505 chan_err(d40c, "Failed to free channel\n");
8d318a50
LW		 2506 spin_unlock_irqrestore(&d40c->lock, flags);
2507}
2508
2509static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2510 dma_addr_t dst,
2511 dma_addr_t src,
2512 size_t size,
2a614340 2513 unsigned long dma_flags)
8d318a50 2514{
95944c6e
RV		 2515 struct scatterlist dst_sg;
2516 struct scatterlist src_sg;
8d318a50 2517
95944c6e
RV		 2518 sg_init_table(&dst_sg, 1);
2519 sg_init_table(&src_sg, 1);
8d318a50 2520
95944c6e
RV		 2521 sg_dma_address(&dst_sg) = dst;
2522 sg_dma_address(&src_sg) = src;
8d318a50 2523
95944c6e
RV		 2524 sg_dma_len(&dst_sg) = size;
2525 sg_dma_len(&src_sg) = size;
8d318a50 2526
cade1d30 2527 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
8d318a50
LW		 2528}
2529
0d688662 2530static struct dma_async_tx_descriptor *
cade1d30
RV		 2531d40_prep_memcpy_sg(struct dma_chan *chan,
2532 struct scatterlist *dst_sg, unsigned int dst_nents,
2533 struct scatterlist *src_sg, unsigned int src_nents,
2534 unsigned long dma_flags)
0d688662
IS		 2535{
2536 if (dst_nents != src_nents)
2537 return NULL;
2538
cade1d30 2539 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
00ac0341
RV		 2540}
2541
f26e03ad
FB		 2542static struct dma_async_tx_descriptor *
2543d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2544 unsigned int sg_len, enum dma_transfer_direction direction,
2545 unsigned long dma_flags, void *context)
8d318a50 2546{
a725dcc0 2547 if (!is_slave_direction(direction))
00ac0341
RV		 2548 return NULL;
2549
cade1d30 2550 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
8d318a50
LW		 2551}
2552
0c842b55
RV		 2553static struct dma_async_tx_descriptor *
2554dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2555 size_t buf_len, size_t period_len,
ec8b5e48
PU		 2556 enum dma_transfer_direction direction, unsigned long flags,
2557 void *context)
0c842b55
RV		 2558{
2559 unsigned int periods = buf_len / period_len;
2560 struct dma_async_tx_descriptor *txd;
2561 struct scatterlist *sg;
2562 int i;
2563
79ca7ec3 2564 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
0c842b55
RV		 2565 for (i = 0; i < periods; i++) {
2566 sg_dma_address(&sg[i]) = dma_addr;
2567 sg_dma_len(&sg[i]) = period_len;
2568 dma_addr += period_len;
2569 }
2570
2571 sg[periods].offset = 0;
fdaf9c4b 2572 sg_dma_len(&sg[periods]) = 0;
0c842b55
RV		 2573 sg[periods].page_link =
2574 ((unsigned long)sg | 0x01) & ~0x02;
2575
2576 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2577 DMA_PREP_INTERRUPT);
2578
2579 kfree(sg);
2580
2581 return txd;
2582}
2583
8d318a50
LW		 2584static enum dma_status d40_tx_status(struct dma_chan *chan,
2585 dma_cookie_t cookie,
2586 struct dma_tx_state *txstate)
2587{
2588 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
96a2af41 2589 enum dma_status ret;
8d318a50 2590
0d0f6b8b 2591 if (d40c->phy_chan == NULL) {
6db5a8ba 2592 chan_err(d40c, "Cannot read status of unallocated channel\n");
0d0f6b8b
JA		 2593 return -EINVAL;
2594 }
2595
96a2af41
RKAL		 2596 ret = dma_cookie_status(chan, cookie, txstate);
2597 if (ret != DMA_SUCCESS)
2598 dma_set_residue(txstate, stedma40_residue(chan));
8d318a50 2599
a5ebca47
JA		 2600 if (d40_is_paused(d40c))
2601 ret = DMA_PAUSED;
8d318a50
LW		 2602
2603 return ret;
2604}
2605
2606static void d40_issue_pending(struct dma_chan *chan)
2607{
2608 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2609 unsigned long flags;
2610
0d0f6b8b 2611 if (d40c->phy_chan == NULL) {
6db5a8ba 2612 chan_err(d40c, "Channel is not allocated!\n");
0d0f6b8b
JA		 2613 return;
2614 }
2615
8d318a50
LW		 2616 spin_lock_irqsave(&d40c->lock, flags);
2617
a8f3067b
PF		 2618 list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
2619
2620 /* Busy means that queued jobs are already being processed */
8d318a50
LW		 2621 if (!d40c->busy)
2622 (void) d40_queue_start(d40c);
2623
2624 spin_unlock_irqrestore(&d40c->lock, flags);
2625}
2626
1bdae6f4
N		 2627static void d40_terminate_all(struct dma_chan *chan)
2628{
2629 unsigned long flags;
2630 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2631 int ret;
2632
2633 spin_lock_irqsave(&d40c->lock, flags);
2634
2635 pm_runtime_get_sync(d40c->base->dev);
2636 ret = d40_channel_execute_command(d40c, D40_DMA_STOP);
2637 if (ret)
2638 chan_err(d40c, "Failed to stop channel\n");
2639
2640 d40_term_all(d40c);
2641 pm_runtime_mark_last_busy(d40c->base->dev);
2642 pm_runtime_put_autosuspend(d40c->base->dev);
2643 if (d40c->busy) {
2644 pm_runtime_mark_last_busy(d40c->base->dev);
2645 pm_runtime_put_autosuspend(d40c->base->dev);
2646 }
2647 d40c->busy = false;
2648
2649 spin_unlock_irqrestore(&d40c->lock, flags);
2650}
2651
98ca5289
RV		 2652static int
2653dma40_config_to_halfchannel(struct d40_chan *d40c,
2654 struct stedma40_half_channel_info *info,
2655 enum dma_slave_buswidth width,
2656 u32 maxburst)
2657{
2658 enum stedma40_periph_data_width addr_width;
2659 int psize;
2660
2661 switch (width) {
2662 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2663 addr_width = STEDMA40_BYTE_WIDTH;
2664 break;
2665 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2666 addr_width = STEDMA40_HALFWORD_WIDTH;
2667 break;
2668 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2669 addr_width = STEDMA40_WORD_WIDTH;
2670 break;
2671 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2672 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2673 break;
2674 default:
2675 dev_err(d40c->base->dev,
2676 "illegal peripheral address width "
2677 "requested (%d)\n",
2678 width);
2679 return -EINVAL;
2680 }
2681
2682 if (chan_is_logical(d40c)) {
2683 if (maxburst >= 16)
2684 psize = STEDMA40_PSIZE_LOG_16;
2685 else if (maxburst >= 8)
2686 psize = STEDMA40_PSIZE_LOG_8;
2687 else if (maxburst >= 4)
2688 psize = STEDMA40_PSIZE_LOG_4;
2689 else
2690 psize = STEDMA40_PSIZE_LOG_1;
2691 } else {
2692 if (maxburst >= 16)
2693 psize = STEDMA40_PSIZE_PHY_16;
2694 else if (maxburst >= 8)
2695 psize = STEDMA40_PSIZE_PHY_8;
2696 else if (maxburst >= 4)
2697 psize = STEDMA40_PSIZE_PHY_4;
2698 else
2699 psize = STEDMA40_PSIZE_PHY_1;
2700 }
2701
2702 info->data_width = addr_width;
2703 info->psize = psize;
2704 info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2705
2706 return 0;
2707}
2708
95e1400f 2709/* Runtime reconfiguration extension */
98ca5289
RV		 2710static int d40_set_runtime_config(struct dma_chan *chan,
2711 struct dma_slave_config *config)
95e1400f
LW		 2712{
2713 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2714 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
98ca5289 2715 enum dma_slave_buswidth src_addr_width, dst_addr_width;
95e1400f 2716 dma_addr_t config_addr;
98ca5289
RV		 2717 u32 src_maxburst, dst_maxburst;
2718 int ret;
2719
2720 src_addr_width = config->src_addr_width;
2721 src_maxburst = config->src_maxburst;
2722 dst_addr_width = config->dst_addr_width;
2723 dst_maxburst = config->dst_maxburst;
95e1400f 2724
db8196df 2725 if (config->direction == DMA_DEV_TO_MEM) {
95e1400f 2726 dma_addr_t dev_addr_rx =
26955c07 2727 d40c->base->plat_data->dev_rx[cfg->dev_type];
95e1400f
LW		 2728
2729 config_addr = config->src_addr;
2730 if (dev_addr_rx)
2731 dev_dbg(d40c->base->dev,
2732 "channel has a pre-wired RX address %08x "
2733 "overriding with %08x\n",
2734 dev_addr_rx, config_addr);
2735 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2736 dev_dbg(d40c->base->dev,
2737 "channel was not configured for peripheral "
2738 "to memory transfer (%d) overriding\n",
2739 cfg->dir);
2740 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2741
98ca5289
RV		 2742 /* Configure the memory side */
2743 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2744 dst_addr_width = src_addr_width;
2745 if (dst_maxburst == 0)
2746 dst_maxburst = src_maxburst;
95e1400f 2747
db8196df 2748 } else if (config->direction == DMA_MEM_TO_DEV) {
95e1400f 2749 dma_addr_t dev_addr_tx =
26955c07 2750 d40c->base->plat_data->dev_tx[cfg->dev_type];
95e1400f
LW		 2751
2752 config_addr = config->dst_addr;
2753 if (dev_addr_tx)
2754 dev_dbg(d40c->base->dev,
2755 "channel has a pre-wired TX address %08x "
2756 "overriding with %08x\n",
2757 dev_addr_tx, config_addr);
2758 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2759 dev_dbg(d40c->base->dev,
2760 "channel was not configured for memory "
2761 "to peripheral transfer (%d) overriding\n",
2762 cfg->dir);
2763 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2764
98ca5289
RV		 2765 /* Configure the memory side */
2766 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2767 src_addr_width = dst_addr_width;
2768 if (src_maxburst == 0)
2769 src_maxburst = dst_maxburst;
95e1400f
LW		 2770 } else {
2771 dev_err(d40c->base->dev,
2772 "unrecognized channel direction %d\n",
2773 config->direction);
98ca5289 2774 return -EINVAL;
95e1400f
LW		 2775 }
2776
98ca5289 2777 if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
95e1400f 2778 dev_err(d40c->base->dev,
98ca5289
RV		 2779 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2780 src_maxburst,
2781 src_addr_width,
2782 dst_maxburst,
2783 dst_addr_width);
2784 return -EINVAL;
95e1400f
LW		 2785 }
2786
92bb6cdb
PF		 2787 if (src_maxburst > 16) {
2788 src_maxburst = 16;
2789 dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;
2790 } else if (dst_maxburst > 16) {
2791 dst_maxburst = 16;
2792 src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;
2793 }
2794
98ca5289
RV		 2795 ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
2796 src_addr_width,
2797 src_maxburst);
2798 if (ret)
2799 return ret;
95e1400f 2800
98ca5289
RV		 2801 ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
2802 dst_addr_width,
2803 dst_maxburst);
2804 if (ret)
2805 return ret;
95e1400f 2806
a59670a4 2807 /* Fill in register values */
724a8577 2808 if (chan_is_logical(d40c))
a59670a4
PF		 2809 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2810 else
2811 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2812 &d40c->dst_def_cfg, false);
2813
95e1400f
LW		 2814 /* These settings will take precedence later */
2815 d40c->runtime_addr = config_addr;
2816 d40c->runtime_direction = config->direction;
2817 dev_dbg(d40c->base->dev,
98ca5289
RV		 2818 "configured channel %s for %s, data width %d/%d, "
2819 "maxburst %d/%d elements, LE, no flow control\n",
95e1400f 2820 dma_chan_name(chan),
db8196df 2821 (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
98ca5289
RV		 2822 src_addr_width, dst_addr_width,
2823 src_maxburst, dst_maxburst);
2824
2825 return 0;
95e1400f
LW		 2826}
2827
05827630
LW		 2828static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2829 unsigned long arg)
8d318a50 2830{
8d318a50
LW		 2831 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2832
0d0f6b8b 2833 if (d40c->phy_chan == NULL) {
6db5a8ba 2834 chan_err(d40c, "Channel is not allocated!\n");
0d0f6b8b
JA		 2835 return -EINVAL;
2836 }
2837
8d318a50
LW		 2838 switch (cmd) {
2839 case DMA_TERMINATE_ALL:
1bdae6f4
N		 2840 d40_terminate_all(chan);
2841 return 0;
8d318a50 2842 case DMA_PAUSE:
86eb5fb6 2843 return d40_pause(d40c);
8d318a50 2844 case DMA_RESUME:
86eb5fb6 2845 return d40_resume(d40c);
95e1400f 2846 case DMA_SLAVE_CONFIG:
98ca5289 2847 return d40_set_runtime_config(chan,
95e1400f 2848 (struct dma_slave_config *) arg);
95e1400f
LW		 2849 default:
2850 break;
8d318a50
LW		 2851 }
2852
2853 /* Other commands are unimplemented */
2854 return -ENXIO;
2855}
2856
2857/* Initialization functions */
2858
2859static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2860 struct d40_chan *chans, int offset,
2861 int num_chans)
2862{
2863 int i = 0;
2864 struct d40_chan *d40c;
2865
2866 INIT_LIST_HEAD(&dma->channels);
2867
2868 for (i = offset; i < offset + num_chans; i++) {
2869 d40c = &chans[i];
2870 d40c->base = base;
2871 d40c->chan.device = dma;
2872
8d318a50
LW		 2873 spin_lock_init(&d40c->lock);
2874
2875 d40c->log_num = D40_PHY_CHAN;
2876
4226dd86 2877 INIT_LIST_HEAD(&d40c->done);
8d318a50
LW		 2878 INIT_LIST_HEAD(&d40c->active);
2879 INIT_LIST_HEAD(&d40c->queue);
a8f3067b 2880 INIT_LIST_HEAD(&d40c->pending_queue);
8d318a50 2881 INIT_LIST_HEAD(&d40c->client);
82babbb3 2882 INIT_LIST_HEAD(&d40c->prepare_queue);
8d318a50 2883
8d318a50
LW		 2884 tasklet_init(&d40c->tasklet, dma_tasklet,
2885 (unsigned long) d40c);
2886
2887 list_add_tail(&d40c->chan.device_node,
2888 &dma->channels);
2889 }
2890}
2891
7ad74a7c
RV		 2892static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2893{
2894 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2895 dev->device_prep_slave_sg = d40_prep_slave_sg;
2896
2897 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2898 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2899
2900 /*
2901 * This controller can only access address at even
2902 * 32bit boundaries, i.e. 2^2
2903 */
2904 dev->copy_align = 2;
2905 }
2906
2907 if (dma_has_cap(DMA_SG, dev->cap_mask))
2908 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2909
0c842b55
RV		 2910 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2911 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2912
7ad74a7c
RV		 2913 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2914 dev->device_free_chan_resources = d40_free_chan_resources;
2915 dev->device_issue_pending = d40_issue_pending;
2916 dev->device_tx_status = d40_tx_status;
2917 dev->device_control = d40_control;
2918 dev->dev = base->dev;
2919}
2920
8d318a50
LW		 2921static int __init d40_dmaengine_init(struct d40_base *base,
2922 int num_reserved_chans)
2923{
2924 int err ;
2925
2926 d40_chan_init(base, &base->dma_slave, base->log_chans,
2927 0, base->num_log_chans);
2928
2929 dma_cap_zero(base->dma_slave.cap_mask);
2930 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
0c842b55 2931 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
8d318a50 2932
7ad74a7c 2933 d40_ops_init(base, &base->dma_slave);
8d318a50
LW		 2934
2935 err = dma_async_device_register(&base->dma_slave);
2936
2937 if (err) {
6db5a8ba 2938 d40_err(base->dev, "Failed to register slave channels\n");
8d318a50
LW		 2939 goto failure1;
2940 }
2941
2942 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
664a57ec 2943 base->num_log_chans, ARRAY_SIZE(dma40_memcpy_channels));
8d318a50
LW		 2944
2945 dma_cap_zero(base->dma_memcpy.cap_mask);
2946 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
7ad74a7c
RV		 2947 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2948
2949 d40_ops_init(base, &base->dma_memcpy);
8d318a50
LW		 2950
2951 err = dma_async_device_register(&base->dma_memcpy);
2952
2953 if (err) {
6db5a8ba
RV		 2954 d40_err(base->dev,
2955 "Failed to regsiter memcpy only channels\n");
8d318a50
LW		 2956 goto failure2;
2957 }
2958
2959 d40_chan_init(base, &base->dma_both, base->phy_chans,
2960 0, num_reserved_chans);
2961
2962 dma_cap_zero(base->dma_both.cap_mask);
2963 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2964 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
7ad74a7c 2965 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
0c842b55 2966 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
7ad74a7c
RV		 2967
2968 d40_ops_init(base, &base->dma_both);
8d318a50
LW		 2969 err = dma_async_device_register(&base->dma_both);
2970
2971 if (err) {
6db5a8ba
RV		 2972 d40_err(base->dev,
2973 "Failed to register logical and physical capable channels\n");
8d318a50
LW		 2974 goto failure3;
2975 }
2976 return 0;
2977failure3:
2978 dma_async_device_unregister(&base->dma_memcpy);
2979failure2:
2980 dma_async_device_unregister(&base->dma_slave);
2981failure1:
2982 return err;
2983}
2984
7fb3e75e
N		 2985/* Suspend resume functionality */
2986#ifdef CONFIG_PM
2987static int dma40_pm_suspend(struct device *dev)
2988{
28c7a19d
N		 2989 struct platform_device *pdev = to_platform_device(dev);
2990 struct d40_base *base = platform_get_drvdata(pdev);
2991 int ret = 0;
7fb3e75e 2992
28c7a19d
N		 2993 if (base->lcpa_regulator)
2994 ret = regulator_disable(base->lcpa_regulator);
2995 return ret;
7fb3e75e
N		 2996}
2997
2998static int dma40_runtime_suspend(struct device *dev)
2999{
3000 struct platform_device *pdev = to_platform_device(dev);
3001 struct d40_base *base = platform_get_drvdata(pdev);
3002
3003 d40_save_restore_registers(base, true);
3004
3005 /* Don't disable/enable clocks for v1 due to HW bugs */
3006 if (base->rev != 1)
3007 writel_relaxed(base->gcc_pwr_off_mask,
3008 base->virtbase + D40_DREG_GCC);
3009
3010 return 0;
3011}
3012
3013static int dma40_runtime_resume(struct device *dev)
3014{
3015 struct platform_device *pdev = to_platform_device(dev);
3016 struct d40_base *base = platform_get_drvdata(pdev);
3017
3018 if (base->initialized)
3019 d40_save_restore_registers(base, false);
3020
3021 writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
3022 base->virtbase + D40_DREG_GCC);
3023 return 0;
3024}
3025
28c7a19d
N		 3026static int dma40_resume(struct device *dev)
3027{
3028 struct platform_device *pdev = to_platform_device(dev);
3029 struct d40_base *base = platform_get_drvdata(pdev);
3030 int ret = 0;
3031
3032 if (base->lcpa_regulator)
3033 ret = regulator_enable(base->lcpa_regulator);
3034
3035 return ret;
3036}
7fb3e75e
N		 3037
3038static const struct dev_pm_ops dma40_pm_ops = {
3039 .suspend = dma40_pm_suspend,
3040 .runtime_suspend = dma40_runtime_suspend,
3041 .runtime_resume = dma40_runtime_resume,
28c7a19d 3042 .resume = dma40_resume,
7fb3e75e
N		 3043};
3044#define DMA40_PM_OPS (&dma40_pm_ops)
3045#else
3046#define DMA40_PM_OPS NULL
3047#endif
3048
8d318a50
LW		 3049/* Initialization functions. */
3050
3051static int __init d40_phy_res_init(struct d40_base *base)
3052{
3053 int i;
3054 int num_phy_chans_avail = 0;
3055 u32 val[2];
3056 int odd_even_bit = -2;
7fb3e75e 3057 int gcc = D40_DREG_GCC_ENA;
8d318a50
LW		 3058
3059 val[0] = readl(base->virtbase + D40_DREG_PRSME);
3060 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
3061
3062 for (i = 0; i < base->num_phy_chans; i++) {
3063 base->phy_res[i].num = i;
3064 odd_even_bit += 2 * ((i % 2) == 0);
3065 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
3066 /* Mark security only channels as occupied */
3067 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
3068 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
7fb3e75e
N		 3069 base->phy_res[i].reserved = true;
3070 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3071 D40_DREG_GCC_SRC);
3072 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3073 D40_DREG_GCC_DST);
3074
3075
8d318a50
LW		 3076 } else {
3077 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
3078 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
7fb3e75e 3079 base->phy_res[i].reserved = false;
8d318a50
LW		 3080 num_phy_chans_avail++;
3081 }
3082 spin_lock_init(&base->phy_res[i].lock);
3083 }
6b7acd84
JA		 3084
3085 /* Mark disabled channels as occupied */
3086 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
f57b407c
RV		 3087 int chan = base->plat_data->disabled_channels[i];
3088
3089 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
3090 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
7fb3e75e
N		 3091 base->phy_res[chan].reserved = true;
3092 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3093 D40_DREG_GCC_SRC);
3094 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3095 D40_DREG_GCC_DST);
f57b407c 3096 num_phy_chans_avail--;
6b7acd84
JA		 3097 }
3098
7407048b
FB		 3099 /* Mark soft_lli channels */
3100 for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {
3101 int chan = base->plat_data->soft_lli_chans[i];
3102
3103 base->phy_res[chan].use_soft_lli = true;
3104 }
3105
8d318a50
LW		 3106 dev_info(base->dev, "%d of %d physical DMA channels available\n",
3107 num_phy_chans_avail, base->num_phy_chans);
3108
3109 /* Verify settings extended vs standard */
3110 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
3111
3112 for (i = 0; i < base->num_phy_chans; i++) {
3113
3114 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
3115 (val[0] & 0x3) != 1)
3116 dev_info(base->dev,
3117 "[%s] INFO: channel %d is misconfigured (%d)\n",
3118 __func__, i, val[0] & 0x3);
3119
3120 val[0] = val[0] >> 2;
3121 }
3122
7fb3e75e
N		 3123 /*
3124 * To keep things simple, Enable all clocks initially.
3125 * The clocks will get managed later post channel allocation.
3126 * The clocks for the event lines on which reserved channels exists
3127 * are not managed here.
3128 */
3129 writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
3130 base->gcc_pwr_off_mask = gcc;
3131
8d318a50
LW		 3132 return num_phy_chans_avail;
3133}
3134
3135static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
3136{
8d318a50
LW		 3137 struct stedma40_platform_data *plat_data;
3138 struct clk *clk = NULL;
3139 void __iomem *virtbase = NULL;
3140 struct resource *res = NULL;
3141 struct d40_base *base = NULL;
3142 int num_log_chans = 0;
3143 int num_phy_chans;
b707c658 3144 int clk_ret = -EINVAL;
8d318a50 3145 int i;
f4b89764
LW		 3146 u32 pid;
3147 u32 cid;
3148 u8 rev;
8d318a50
LW		 3149
3150 clk = clk_get(&pdev->dev, NULL);
8d318a50 3151 if (IS_ERR(clk)) {
6db5a8ba 3152 d40_err(&pdev->dev, "No matching clock found\n");
8d318a50
LW		 3153 goto failure;
3154 }
3155
b707c658
UH		 3156 clk_ret = clk_prepare_enable(clk);
3157 if (clk_ret) {
3158 d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
3159 goto failure;
3160 }
8d318a50
LW		 3161
3162 /* Get IO for DMAC base address */
3163 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3164 if (!res)
3165 goto failure;
3166
3167 if (request_mem_region(res->start, resource_size(res),
3168 D40_NAME " I/O base") == NULL)
3169 goto failure;
3170
3171 virtbase = ioremap(res->start, resource_size(res));
3172 if (!virtbase)
3173 goto failure;
3174
f4b89764
LW		 3175 /* This is just a regular AMBA PrimeCell ID actually */
3176 for (pid = 0, i = 0; i < 4; i++)
3177 pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
3178 & 255) << (i * 8);
3179 for (cid = 0, i = 0; i < 4; i++)
3180 cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
3181 & 255) << (i * 8);
8d318a50 3182
f4b89764
LW		 3183 if (cid != AMBA_CID) {
3184 d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
3185 goto failure;
3186 }
3187 if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
6db5a8ba 3188 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
f4b89764
LW		 3189 AMBA_MANF_BITS(pid),
3190 AMBA_VENDOR_ST);
8d318a50
LW		 3191 goto failure;
3192 }
f4b89764
LW		 3193 /*
3194 * HW revision:
3195 * DB8500ed has revision 0
3196 * ? has revision 1
3197 * DB8500v1 has revision 2
3198 * DB8500v2 has revision 3
47db92f4
GB		 3199 * AP9540v1 has revision 4
3200 * DB8540v1 has revision 4
f4b89764
LW		 3201 */
3202 rev = AMBA_REV_BITS(pid);
3ae0267f 3203
47db92f4
GB		 3204 plat_data = pdev->dev.platform_data;
3205
8d318a50 3206 /* The number of physical channels on this HW */
47db92f4
GB		 3207 if (plat_data->num_of_phy_chans)
3208 num_phy_chans = plat_data->num_of_phy_chans;
3209 else
3210 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
8d318a50 3211
47db92f4
GB		 3212 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x with %d physical channels\n",
3213 rev, res->start, num_phy_chans);
8d318a50 3214
1bdae6f4
N		 3215 if (rev < 2) {
3216 d40_err(&pdev->dev, "hardware revision: %d is not supported",
3217 rev);
3218 goto failure;
3219 }
3220
8d318a50
LW		 3221 /* Count the number of logical channels in use */
3222 for (i = 0; i < plat_data->dev_len; i++)
3223 if (plat_data->dev_rx[i] != 0)
3224 num_log_chans++;
3225
3226 for (i = 0; i < plat_data->dev_len; i++)
3227 if (plat_data->dev_tx[i] != 0)
3228 num_log_chans++;
3229
3230 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
664a57ec 3231 (num_phy_chans + num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) *
8d318a50
LW		 3232 sizeof(struct d40_chan), GFP_KERNEL);
3233
3234 if (base == NULL) {
6db5a8ba 3235 d40_err(&pdev->dev, "Out of memory\n");
8d318a50
LW		 3236 goto failure;
3237 }
3238
3ae0267f 3239 base->rev = rev;
8d318a50
LW		 3240 base->clk = clk;
3241 base->num_phy_chans = num_phy_chans;
3242 base->num_log_chans = num_log_chans;
3243 base->phy_start = res->start;
3244 base->phy_size = resource_size(res);
3245 base->virtbase = virtbase;
3246 base->plat_data = plat_data;
3247 base->dev = &pdev->dev;
3248 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
3249 base->log_chans = &base->phy_chans[num_phy_chans];
3250
3cb645dc
TL		 3251 if (base->plat_data->num_of_phy_chans == 14) {
3252 base->gen_dmac.backup = d40_backup_regs_v4b;
3253 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;
3254 base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;
3255 base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;
3256 base->gen_dmac.realtime_en = D40_DREG_CRSEG1;
3257 base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;
3258 base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;
3259 base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;
3260 base->gen_dmac.il = il_v4b;
3261 base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);
3262 base->gen_dmac.init_reg = dma_init_reg_v4b;
3263 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);
3264 } else {
3265 if (base->rev >= 3) {
3266 base->gen_dmac.backup = d40_backup_regs_v4a;
3267 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;
3268 }
3269 base->gen_dmac.interrupt_en = D40_DREG_PCMIS;
3270 base->gen_dmac.interrupt_clear = D40_DREG_PCICR;
3271 base->gen_dmac.realtime_en = D40_DREG_RSEG1;
3272 base->gen_dmac.realtime_clear = D40_DREG_RCEG1;
3273 base->gen_dmac.high_prio_en = D40_DREG_PSEG1;
3274 base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;
3275 base->gen_dmac.il = il_v4a;
3276 base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);
3277 base->gen_dmac.init_reg = dma_init_reg_v4a;
3278 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
3279 }
3280
8d318a50
LW		 3281 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
3282 GFP_KERNEL);
3283 if (!base->phy_res)
3284 goto failure;
3285
3286 base->lookup_phy_chans = kzalloc(num_phy_chans *
3287 sizeof(struct d40_chan *),
3288 GFP_KERNEL);
3289 if (!base->lookup_phy_chans)
3290 goto failure;
3291
664a57ec 3292 if (num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) {
8d318a50
LW		 3293 /*
3294 * The max number of logical channels are event lines for all
3295 * src devices and dst devices
3296 */
3297 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
3298 sizeof(struct d40_chan *),
3299 GFP_KERNEL);
3300 if (!base->lookup_log_chans)
3301 goto failure;
3302 }
698e4732 3303
7fb3e75e
N		 3304 base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
3305 sizeof(d40_backup_regs_chan),
8d318a50 3306 GFP_KERNEL);
7fb3e75e
N		 3307 if (!base->reg_val_backup_chan)
3308 goto failure;
3309
3310 base->lcla_pool.alloc_map =
3311 kzalloc(num_phy_chans * sizeof(struct d40_desc *)
3312 * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
8d318a50
LW		 3313 if (!base->lcla_pool.alloc_map)
3314 goto failure;
3315
c675b1b4
JA		 3316 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
3317 0, SLAB_HWCACHE_ALIGN,
3318 NULL);
3319 if (base->desc_slab == NULL)
3320 goto failure;
3321
8d318a50
LW		 3322 return base;
3323
3324failure:
b707c658
UH		 3325 if (!clk_ret)
3326 clk_disable_unprepare(clk);
3327 if (!IS_ERR(clk))
8d318a50 3328 clk_put(clk);
8d318a50
LW		 3329 if (virtbase)
3330 iounmap(virtbase);
3331 if (res)
3332 release_mem_region(res->start,
3333 resource_size(res));
3334 if (virtbase)
3335 iounmap(virtbase);
3336
3337 if (base) {
3338 kfree(base->lcla_pool.alloc_map);
1bdae6f4 3339 kfree(base->reg_val_backup_chan);
8d318a50
LW		 3340 kfree(base->lookup_log_chans);
3341 kfree(base->lookup_phy_chans);
3342 kfree(base->phy_res);
3343 kfree(base);
3344 }
3345
3346 return NULL;
3347}
3348
3349static void __init d40_hw_init(struct d40_base *base)
3350{
3351
8d318a50
LW		 3352 int i;
3353 u32 prmseo[2] = {0, 0};
3354 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3355 u32 pcmis = 0;
3356 u32 pcicr = 0;
3cb645dc
TL		 3357 struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;
3358 u32 reg_size = base->gen_dmac.init_reg_size;
8d318a50 3359
3cb645dc 3360 for (i = 0; i < reg_size; i++)
8d318a50
LW		 3361 writel(dma_init_reg[i].val,
3362 base->virtbase + dma_init_reg[i].reg);
3363
3364 /* Configure all our dma channels to default settings */
3365 for (i = 0; i < base->num_phy_chans; i++) {
3366
3367 activeo[i % 2] = activeo[i % 2] << 2;
3368
3369 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
3370 == D40_ALLOC_PHY) {
3371 activeo[i % 2] |= 3;
3372 continue;
3373 }
3374
3375 /* Enable interrupt # */
3376 pcmis = (pcmis << 1) | 1;
3377
3378 /* Clear interrupt # */
3379 pcicr = (pcicr << 1) | 1;
3380
3381 /* Set channel to physical mode */
3382 prmseo[i % 2] = prmseo[i % 2] << 2;
3383 prmseo[i % 2] |= 1;
3384
3385 }
3386
3387 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
3388 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
3389 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
3390 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
3391
3392 /* Write which interrupt to enable */
3cb645dc 3393 writel(pcmis, base->virtbase + base->gen_dmac.interrupt_en);
8d318a50
LW		 3394
3395 /* Write which interrupt to clear */
3cb645dc 3396 writel(pcicr, base->virtbase + base->gen_dmac.interrupt_clear);
8d318a50 3397
3cb645dc
TL		 3398 /* These are __initdata and cannot be accessed after init */
3399 base->gen_dmac.init_reg = NULL;
3400 base->gen_dmac.init_reg_size = 0;
8d318a50
LW		 3401}
3402
508849ad
LW		 3403static int __init d40_lcla_allocate(struct d40_base *base)
3404{
026cbc42 3405 struct d40_lcla_pool *pool = &base->lcla_pool;
508849ad
LW		 3406 unsigned long *page_list;
3407 int i, j;
3408 int ret = 0;
3409
3410 /*
3411 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3412 * To full fill this hardware requirement without wasting 256 kb
3413 * we allocate pages until we get an aligned one.
3414 */
3415 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
3416 GFP_KERNEL);
3417
3418 if (!page_list) {
3419 ret = -ENOMEM;
3420 goto failure;
3421 }
3422
3423 /* Calculating how many pages that are required */
3424 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
3425
3426 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
3427 page_list[i] = __get_free_pages(GFP_KERNEL,
3428 base->lcla_pool.pages);
3429 if (!page_list[i]) {
3430
6db5a8ba
RV		 3431 d40_err(base->dev, "Failed to allocate %d pages.\n",
3432 base->lcla_pool.pages);
508849ad
LW		 3433
3434 for (j = 0; j < i; j++)
3435 free_pages(page_list[j], base->lcla_pool.pages);
3436 goto failure;
3437 }
3438
3439 if ((virt_to_phys((void *)page_list[i]) &
3440 (LCLA_ALIGNMENT - 1)) == 0)
3441 break;
3442 }
3443
3444 for (j = 0; j < i; j++)
3445 free_pages(page_list[j], base->lcla_pool.pages);
3446
3447 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
3448 base->lcla_pool.base = (void *)page_list[i];
3449 } else {
767a9675
JA		 3450 /*
3451 * After many attempts and no succees with finding the correct
3452 * alignment, try with allocating a big buffer.
3453 */
508849ad
LW		 3454 dev_warn(base->dev,
3455 "[%s] Failed to get %d pages @ 18 bit align.\n",
3456 __func__, base->lcla_pool.pages);
3457 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
3458 base->num_phy_chans +
3459 LCLA_ALIGNMENT,
3460 GFP_KERNEL);
3461 if (!base->lcla_pool.base_unaligned) {
3462 ret = -ENOMEM;
3463 goto failure;
3464 }
3465
3466 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
3467 LCLA_ALIGNMENT);
3468 }
3469
026cbc42
RV		 3470 pool->dma_addr = dma_map_single(base->dev, pool->base,
3471 SZ_1K * base->num_phy_chans,
3472 DMA_TO_DEVICE);
3473 if (dma_mapping_error(base->dev, pool->dma_addr)) {
3474 pool->dma_addr = 0;
3475 ret = -ENOMEM;
3476 goto failure;
3477 }
3478
508849ad
LW		 3479 writel(virt_to_phys(base->lcla_pool.base),
3480 base->virtbase + D40_DREG_LCLA);
3481failure:
3482 kfree(page_list);
3483 return ret;
3484}
3485
8d318a50
LW		 3486static int __init d40_probe(struct platform_device *pdev)
3487{
3488 int err;
3489 int ret = -ENOENT;
3490 struct d40_base *base;
3491 struct resource *res = NULL;
3492 int num_reserved_chans;
3493 u32 val;
3494
3495 base = d40_hw_detect_init(pdev);
3496
3497 if (!base)
3498 goto failure;
3499
3500 num_reserved_chans = d40_phy_res_init(base);
3501
3502 platform_set_drvdata(pdev, base);
3503
3504 spin_lock_init(&base->interrupt_lock);
3505 spin_lock_init(&base->execmd_lock);
3506
3507 /* Get IO for logical channel parameter address */
3508 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
3509 if (!res) {
3510 ret = -ENOENT;
6db5a8ba 3511 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
8d318a50
LW		 3512 goto failure;
3513 }
3514 base->lcpa_size = resource_size(res);
3515 base->phy_lcpa = res->start;
3516
3517 if (request_mem_region(res->start, resource_size(res),
3518 D40_NAME " I/O lcpa") == NULL) {
3519 ret = -EBUSY;
6db5a8ba
RV		 3520 d40_err(&pdev->dev,
3521 "Failed to request LCPA region 0x%x-0x%x\n",
3522 res->start, res->end);
8d318a50
LW		 3523 goto failure;
3524 }
3525
3526 /* We make use of ESRAM memory for this. */
3527 val = readl(base->virtbase + D40_DREG_LCPA);
3528 if (res->start != val && val != 0) {
3529 dev_warn(&pdev->dev,
3530 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
3531 __func__, val, res->start);
3532 } else
3533 writel(res->start, base->virtbase + D40_DREG_LCPA);
3534
3535 base->lcpa_base = ioremap(res->start, resource_size(res));
3536 if (!base->lcpa_base) {
3537 ret = -ENOMEM;
6db5a8ba 3538 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
8d318a50
LW		 3539 goto failure;
3540 }
28c7a19d
N		 3541 /* If lcla has to be located in ESRAM we don't need to allocate */
3542 if (base->plat_data->use_esram_lcla) {
3543 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
3544 "lcla_esram");
3545 if (!res) {
3546 ret = -ENOENT;
3547 d40_err(&pdev->dev,
3548 "No \"lcla_esram\" memory resource\n");
3549 goto failure;
3550 }
3551 base->lcla_pool.base = ioremap(res->start,
3552 resource_size(res));
3553 if (!base->lcla_pool.base) {
3554 ret = -ENOMEM;
3555 d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
3556 goto failure;
3557 }
3558 writel(res->start, base->virtbase + D40_DREG_LCLA);
8d318a50 3559
28c7a19d
N		 3560 } else {
3561 ret = d40_lcla_allocate(base);
3562 if (ret) {
3563 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
3564 goto failure;
3565 }
8d318a50
LW		 3566 }
3567
3568 spin_lock_init(&base->lcla_pool.lock);
3569
8d318a50
LW		 3570 base->irq = platform_get_irq(pdev, 0);
3571
3572 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
8d318a50 3573 if (ret) {
6db5a8ba 3574 d40_err(&pdev->dev, "No IRQ defined\n");
8d318a50
LW		 3575 goto failure;
3576 }
3577
7fb3e75e
N		 3578 pm_runtime_irq_safe(base->dev);
3579 pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
3580 pm_runtime_use_autosuspend(base->dev);
3581 pm_runtime_enable(base->dev);
3582 pm_runtime_resume(base->dev);
28c7a19d
N		 3583
3584 if (base->plat_data->use_esram_lcla) {
3585
3586 base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
3587 if (IS_ERR(base->lcpa_regulator)) {
3588 d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
3589 base->lcpa_regulator = NULL;
3590 goto failure;
3591 }
3592
3593 ret = regulator_enable(base->lcpa_regulator);
3594 if (ret) {
3595 d40_err(&pdev->dev,
3596 "Failed to enable lcpa_regulator\n");
3597 regulator_put(base->lcpa_regulator);
3598 base->lcpa_regulator = NULL;
3599 goto failure;
3600 }
3601 }
3602
7fb3e75e 3603 base->initialized = true;
8d318a50
LW		 3604 err = d40_dmaengine_init(base, num_reserved_chans);
3605 if (err)
3606 goto failure;
3607
b96710e5
PF		 3608 base->dev->dma_parms = &base->dma_parms;
3609 err = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);
3610 if (err) {
3611 d40_err(&pdev->dev, "Failed to set dma max seg size\n");
3612 goto failure;
3613 }
3614
8d318a50
LW		 3615 d40_hw_init(base);
3616
3617 dev_info(base->dev, "initialized\n");
3618 return 0;
3619
3620failure:
3621 if (base) {
c675b1b4
JA		 3622 if (base->desc_slab)
3623 kmem_cache_destroy(base->desc_slab);
8d318a50
LW		 3624 if (base->virtbase)
3625 iounmap(base->virtbase);
026cbc42 3626
28c7a19d
N		 3627 if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
3628 iounmap(base->lcla_pool.base);
3629 base->lcla_pool.base = NULL;
3630 }
3631
026cbc42
RV		 3632 if (base->lcla_pool.dma_addr)
3633 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
3634 SZ_1K * base->num_phy_chans,
3635 DMA_TO_DEVICE);
3636
508849ad
LW		 3637 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3638 free_pages((unsigned long)base->lcla_pool.base,
3639 base->lcla_pool.pages);
767a9675
JA		 3640
3641 kfree(base->lcla_pool.base_unaligned);
3642
8d318a50
LW		 3643 if (base->phy_lcpa)
3644 release_mem_region(base->phy_lcpa,
3645 base->lcpa_size);
3646 if (base->phy_start)
3647 release_mem_region(base->phy_start,
3648 base->phy_size);
3649 if (base->clk) {
da2ac56a 3650 clk_disable_unprepare(base->clk);
8d318a50
LW		 3651 clk_put(base->clk);
3652 }
3653
28c7a19d
N		 3654 if (base->lcpa_regulator) {
3655 regulator_disable(base->lcpa_regulator);
3656 regulator_put(base->lcpa_regulator);
3657 }
3658
8d318a50
LW		 3659 kfree(base->lcla_pool.alloc_map);
3660 kfree(base->lookup_log_chans);
3661 kfree(base->lookup_phy_chans);
3662 kfree(base->phy_res);
3663 kfree(base);
3664 }
3665
6db5a8ba 3666 d40_err(&pdev->dev, "probe failed\n");
8d318a50
LW		 3667 return ret;
3668}
3669
3670static struct platform_driver d40_driver = {
3671 .driver = {
3672 .owner = THIS_MODULE,
3673 .name = D40_NAME,
7fb3e75e 3674 .pm = DMA40_PM_OPS,
8d318a50
LW		 3675 },
3676};
3677
cb9ab2d8 3678static int __init stedma40_init(void)
8d318a50
LW		 3679{
3680 return platform_driver_probe(&d40_driver, d40_probe);
3681}
a0eb221a 3682subsys_initcall(stedma40_init);
ubuntu-bionic-kernel mirror