]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/dma/ste_dma40.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: ste_dma40: Rectify incorrect configuration validation checking
[mirror_ubuntu-bionic-kernel.git] / drivers / dma / ste_dma40.c
CommitLineData
8d318a50 1/*
d49278e3
PF		 2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
661385f9 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
767a9675 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
8d318a50 6 * License terms: GNU General Public License (GPL) version 2
8d318a50
LW		 7 */
8
b7f080cf 9#include <linux/dma-mapping.h>
8d318a50
LW		 10#include <linux/kernel.h>
11#include <linux/slab.h>
f492b210 12#include <linux/export.h>
8d318a50
LW		 13#include <linux/dmaengine.h>
14#include <linux/platform_device.h>
15#include <linux/clk.h>
16#include <linux/delay.h>
7fb3e75e
N		 17#include <linux/pm.h>
18#include <linux/pm_runtime.h>
698e4732 19#include <linux/err.h>
f4b89764 20#include <linux/amba/bus.h>
15e4b78d 21#include <linux/regulator/consumer.h>
865fab60 22#include <linux/platform_data/dma-ste-dma40.h>
8d318a50 23
d2ebfb33 24#include "dmaengine.h"
8d318a50
LW		 25#include "ste_dma40_ll.h"
26
27#define D40_NAME "dma40"
28
29#define D40_PHY_CHAN -1
30
31/* For masking out/in 2 bit channel positions */
32#define D40_CHAN_POS(chan) (2 * (chan / 2))
33#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
34
35/* Maximum iterations taken before giving up suspending a channel */
36#define D40_SUSPEND_MAX_IT 500
37
7fb3e75e
N		 38/* Milliseconds */
39#define DMA40_AUTOSUSPEND_DELAY 100
40
508849ad
LW		 41/* Hardware requirement on LCLA alignment */
42#define LCLA_ALIGNMENT 0x40000
698e4732
JA		 43
44/* Max number of links per event group */
45#define D40_LCLA_LINK_PER_EVENT_GRP 128
46#define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
47
508849ad
LW		 48/* Attempts before giving up to trying to get pages that are aligned */
49#define MAX_LCLA_ALLOC_ATTEMPTS 256
50
51/* Bit markings for allocation map */
8d318a50
LW		 52#define D40_ALLOC_FREE (1 << 31)
53#define D40_ALLOC_PHY (1 << 30)
54#define D40_ALLOC_LOG_FREE 0
55
3cb645dc
TL		 56#define MAX(a, b) (((a) < (b)) ? (b) : (a))
57
664a57ec 58/* Reserved event lines for memcpy only. */
a2acaa21
LW		 59#define DB8500_DMA_MEMCPY_EV_0 51
60#define DB8500_DMA_MEMCPY_EV_1 56
61#define DB8500_DMA_MEMCPY_EV_2 57
62#define DB8500_DMA_MEMCPY_EV_3 58
63#define DB8500_DMA_MEMCPY_EV_4 59
64#define DB8500_DMA_MEMCPY_EV_5 60
65
66static int dma40_memcpy_channels[] = {
67 DB8500_DMA_MEMCPY_EV_0,
68 DB8500_DMA_MEMCPY_EV_1,
69 DB8500_DMA_MEMCPY_EV_2,
70 DB8500_DMA_MEMCPY_EV_3,
71 DB8500_DMA_MEMCPY_EV_4,
72 DB8500_DMA_MEMCPY_EV_5,
73};
664a57ec 74
29027a1e
LJ		 75/* Default configuration for physcial memcpy */
76struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
77 .mode = STEDMA40_MODE_PHYSICAL,
78 .dir = STEDMA40_MEM_TO_MEM,
79
80 .src_info.data_width = STEDMA40_BYTE_WIDTH,
81 .src_info.psize = STEDMA40_PSIZE_PHY_1,
82 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
83
84 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
85 .dst_info.psize = STEDMA40_PSIZE_PHY_1,
86 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
87};
88
89/* Default configuration for logical memcpy */
90struct stedma40_chan_cfg dma40_memcpy_conf_log = {
91 .mode = STEDMA40_MODE_LOGICAL,
92 .dir = STEDMA40_MEM_TO_MEM,
93
94 .src_info.data_width = STEDMA40_BYTE_WIDTH,
95 .src_info.psize = STEDMA40_PSIZE_LOG_1,
96 .src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
97
98 .dst_info.data_width = STEDMA40_BYTE_WIDTH,
99 .dst_info.psize = STEDMA40_PSIZE_LOG_1,
100 .dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
101};
102
8d318a50
LW		 103/**
104 * enum 40_command - The different commands and/or statuses.
105 *
106 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
107 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
108 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
109 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
110 */
111enum d40_command {
112 D40_DMA_STOP = 0,
113 D40_DMA_RUN = 1,
114 D40_DMA_SUSPEND_REQ = 2,
115 D40_DMA_SUSPENDED = 3
116};
117
1bdae6f4
N		 118/*
119 * enum d40_events - The different Event Enables for the event lines.
120 *
121 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
122 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
123 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
124 * @D40_ROUND_EVENTLINE: Status check for event line.
125 */
126
127enum d40_events {
128 D40_DEACTIVATE_EVENTLINE = 0,
129 D40_ACTIVATE_EVENTLINE = 1,
130 D40_SUSPEND_REQ_EVENTLINE = 2,
131 D40_ROUND_EVENTLINE = 3
132};
133
7fb3e75e
N		 134/*
135 * These are the registers that has to be saved and later restored
136 * when the DMA hw is powered off.
137 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
138 */
139static u32 d40_backup_regs[] = {
140 D40_DREG_LCPA,
141 D40_DREG_LCLA,
142 D40_DREG_PRMSE,
143 D40_DREG_PRMSO,
144 D40_DREG_PRMOE,
145 D40_DREG_PRMOO,
146};
147
148#define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
149
3cb645dc
TL		 150/*
151 * since 9540 and 8540 has the same HW revision
152 * use v4a for 9540 or ealier
153 * use v4b for 8540 or later
154 * HW revision:
155 * DB8500ed has revision 0
156 * DB8500v1 has revision 2
157 * DB8500v2 has revision 3
158 * AP9540v1 has revision 4
159 * DB8540v1 has revision 4
160 * TODO: Check if all these registers have to be saved/restored on dma40 v4a
161 */
162static u32 d40_backup_regs_v4a[] = {
7fb3e75e
N		 163 D40_DREG_PSEG1,
164 D40_DREG_PSEG2,
165 D40_DREG_PSEG3,
166 D40_DREG_PSEG4,
167 D40_DREG_PCEG1,
168 D40_DREG_PCEG2,
169 D40_DREG_PCEG3,
170 D40_DREG_PCEG4,
171 D40_DREG_RSEG1,
172 D40_DREG_RSEG2,
173 D40_DREG_RSEG3,
174 D40_DREG_RSEG4,
175 D40_DREG_RCEG1,
176 D40_DREG_RCEG2,
177 D40_DREG_RCEG3,
178 D40_DREG_RCEG4,
179};
180
3cb645dc
TL		 181#define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
182
183static u32 d40_backup_regs_v4b[] = {
184 D40_DREG_CPSEG1,
185 D40_DREG_CPSEG2,
186 D40_DREG_CPSEG3,
187 D40_DREG_CPSEG4,
188 D40_DREG_CPSEG5,
189 D40_DREG_CPCEG1,
190 D40_DREG_CPCEG2,
191 D40_DREG_CPCEG3,
192 D40_DREG_CPCEG4,
193 D40_DREG_CPCEG5,
194 D40_DREG_CRSEG1,
195 D40_DREG_CRSEG2,
196 D40_DREG_CRSEG3,
197 D40_DREG_CRSEG4,
198 D40_DREG_CRSEG5,
199 D40_DREG_CRCEG1,
200 D40_DREG_CRCEG2,
201 D40_DREG_CRCEG3,
202 D40_DREG_CRCEG4,
203 D40_DREG_CRCEG5,
204};
205
206#define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
7fb3e75e
N		 207
208static u32 d40_backup_regs_chan[] = {
209 D40_CHAN_REG_SSCFG,
210 D40_CHAN_REG_SSELT,
211 D40_CHAN_REG_SSPTR,
212 D40_CHAN_REG_SSLNK,
213 D40_CHAN_REG_SDCFG,
214 D40_CHAN_REG_SDELT,
215 D40_CHAN_REG_SDPTR,
216 D40_CHAN_REG_SDLNK,
217};
218
3cb645dc
TL		 219/**
220 * struct d40_interrupt_lookup - lookup table for interrupt handler
221 *
222 * @src: Interrupt mask register.
223 * @clr: Interrupt clear register.
224 * @is_error: true if this is an error interrupt.
225 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
226 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
227 */
228struct d40_interrupt_lookup {
229 u32 src;
230 u32 clr;
231 bool is_error;
232 int offset;
233};
234
235
236static struct d40_interrupt_lookup il_v4a[] = {
237 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
238 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
239 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
240 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
241 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
242 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
243 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
244 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
245 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
246 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
247};
248
249static struct d40_interrupt_lookup il_v4b[] = {
250 {D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false, 0},
251 {D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},
252 {D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},
253 {D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},
254 {D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},
255 {D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true, 0},
256 {D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true, 32},
257 {D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true, 64},
258 {D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true, 96},
259 {D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true, 128},
260 {D40_DREG_CPCTIS, D40_DREG_CPCICR, false, D40_PHY_CHAN},
261 {D40_DREG_CPCEIS, D40_DREG_CPCICR, true, D40_PHY_CHAN},
262};
263
264/**
265 * struct d40_reg_val - simple lookup struct
266 *
267 * @reg: The register.
268 * @val: The value that belongs to the register in reg.
269 */
270struct d40_reg_val {
271 unsigned int reg;
272 unsigned int val;
273};
274
275static __initdata struct d40_reg_val dma_init_reg_v4a[] = {
276 /* Clock every part of the DMA block from start */
277 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
278
279 /* Interrupts on all logical channels */
280 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
281 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
282 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
283 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
284 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
285 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
286 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
287 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
288 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
289 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
290 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
291 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
292};
293static __initdata struct d40_reg_val dma_init_reg_v4b[] = {
294 /* Clock every part of the DMA block from start */
295 { .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
296
297 /* Interrupts on all logical channels */
298 { .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},
299 { .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},
300 { .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},
301 { .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},
302 { .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},
303 { .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},
304 { .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},
305 { .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},
306 { .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},
307 { .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},
308 { .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},
309 { .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},
310 { .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},
311 { .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},
312 { .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}
313};
314
8d318a50
LW		 315/**
316 * struct d40_lli_pool - Structure for keeping LLIs in memory
317 *
318 * @base: Pointer to memory area when the pre_alloc_lli's are not large
319 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
320 * pre_alloc_lli is used.
b00f938c 321 * @dma_addr: DMA address, if mapped
8d318a50
LW		 322 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
323 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
324 * one buffer to one buffer.
325 */
326struct d40_lli_pool {
327 void *base;
508849ad 328 int size;
b00f938c 329 dma_addr_t dma_addr;
8d318a50 330 /* Space for dst and src, plus an extra for padding */
508849ad 331 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
8d318a50
LW		 332};
333
334/**
335 * struct d40_desc - A descriptor is one DMA job.
336 *
337 * @lli_phy: LLI settings for physical channel. Both src and dst=
338 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
339 * lli_len equals one.
340 * @lli_log: Same as above but for logical channels.
341 * @lli_pool: The pool with two entries pre-allocated.
941b77a3 342 * @lli_len: Number of llis of current descriptor.
25985edc 343 * @lli_current: Number of transferred llis.
698e4732 344 * @lcla_alloc: Number of LCLA entries allocated.
8d318a50
LW		 345 * @txd: DMA engine struct. Used for among other things for communication
346 * during a transfer.
347 * @node: List entry.
8d318a50 348 * @is_in_client_list: true if the client owns this descriptor.
7fb3e75e 349 * @cyclic: true if this is a cyclic job
8d318a50
LW		 350 *
351 * This descriptor is used for both logical and physical transfers.
352 */
8d318a50
LW		 353struct d40_desc {
354 /* LLI physical */
355 struct d40_phy_lli_bidir lli_phy;
356 /* LLI logical */
357 struct d40_log_lli_bidir lli_log;
358
359 struct d40_lli_pool lli_pool;
941b77a3 360 int lli_len;
698e4732
JA		 361 int lli_current;
362 int lcla_alloc;
8d318a50
LW		 363
364 struct dma_async_tx_descriptor txd;
365 struct list_head node;
366
8d318a50 367 bool is_in_client_list;
0c842b55 368 bool cyclic;
8d318a50
LW		 369};
370
371/**
372 * struct d40_lcla_pool - LCLA pool settings and data.
373 *
508849ad
LW		 374 * @base: The virtual address of LCLA. 18 bit aligned.
375 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
376 * This pointer is only there for clean-up on error.
377 * @pages: The number of pages needed for all physical channels.
378 * Only used later for clean-up on error
8d318a50 379 * @lock: Lock to protect the content in this struct.
698e4732 380 * @alloc_map: big map over which LCLA entry is own by which job.
8d318a50
LW		 381 */
382struct d40_lcla_pool {
383 void *base;
026cbc42 384 dma_addr_t dma_addr;
508849ad
LW		 385 void *base_unaligned;
386 int pages;
8d318a50 387 spinlock_t lock;
698e4732 388 struct d40_desc **alloc_map;
8d318a50
LW		 389};
390
391/**
392 * struct d40_phy_res - struct for handling eventlines mapped to physical
393 * channels.
394 *
395 * @lock: A lock protection this entity.
7fb3e75e 396 * @reserved: True if used by secure world or otherwise.
8d318a50
LW		 397 * @num: The physical channel number of this entity.
398 * @allocated_src: Bit mapped to show which src event line's are mapped to
399 * this physical channel. Can also be free or physically allocated.
400 * @allocated_dst: Same as for src but is dst.
401 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
767a9675 402 * event line number.
7407048b 403 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
8d318a50
LW		 404 */
405struct d40_phy_res {
406 spinlock_t lock;
7fb3e75e 407 bool reserved;
8d318a50
LW		 408 int num;
409 u32 allocated_src;
410 u32 allocated_dst;
7407048b 411 bool use_soft_lli;
8d318a50
LW		 412};
413
414struct d40_base;
415
416/**
417 * struct d40_chan - Struct that describes a channel.
418 *
419 * @lock: A spinlock to protect this struct.
420 * @log_num: The logical number, if any of this channel.
8d318a50
LW		 421 * @pending_tx: The number of pending transfers. Used between interrupt handler
422 * and tasklet.
423 * @busy: Set to true when transfer is ongoing on this channel.
2a614340
JA		 424 * @phy_chan: Pointer to physical channel which this instance runs on. If this
425 * point is NULL, then the channel is not allocated.
8d318a50
LW		 426 * @chan: DMA engine handle.
427 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
428 * transfer and call client callback.
429 * @client: Cliented owned descriptor list.
da063d26 430 * @pending_queue: Submitted jobs, to be issued by issue_pending()
8d318a50 431 * @active: Active descriptor.
4226dd86 432 * @done: Completed jobs
8d318a50 433 * @queue: Queued jobs.
82babbb3 434 * @prepare_queue: Prepared jobs.
8d318a50 435 * @dma_cfg: The client configuration of this dma channel.
ce2ca125 436 * @configured: whether the dma_cfg configuration is valid
8d318a50
LW		 437 * @base: Pointer to the device instance struct.
438 * @src_def_cfg: Default cfg register setting for src.
439 * @dst_def_cfg: Default cfg register setting for dst.
440 * @log_def: Default logical channel settings.
8d318a50 441 * @lcpa: Pointer to dst and src lcpa settings.
ae752bf4 442 * @runtime_addr: runtime configured address.
443 * @runtime_direction: runtime configured direction.
8d318a50
LW		 444 *
445 * This struct can either "be" a logical or a physical channel.
446 */
447struct d40_chan {
448 spinlock_t lock;
449 int log_num;
8d318a50
LW		 450 int pending_tx;
451 bool busy;
452 struct d40_phy_res *phy_chan;
453 struct dma_chan chan;
454 struct tasklet_struct tasklet;
455 struct list_head client;
a8f3067b 456 struct list_head pending_queue;
8d318a50 457 struct list_head active;
4226dd86 458 struct list_head done;
8d318a50 459 struct list_head queue;
82babbb3 460 struct list_head prepare_queue;
8d318a50 461 struct stedma40_chan_cfg dma_cfg;
ce2ca125 462 bool configured;
8d318a50
LW		 463 struct d40_base *base;
464 /* Default register configurations */
465 u32 src_def_cfg;
466 u32 dst_def_cfg;
467 struct d40_def_lcsp log_def;
8d318a50 468 struct d40_log_lli_full *lcpa;
95e1400f
LW		 469 /* Runtime reconfiguration */
470 dma_addr_t runtime_addr;
db8196df 471 enum dma_transfer_direction runtime_direction;
8d318a50
LW		 472};
473
3cb645dc
TL		 474/**
475 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
476 * controller
477 *
478 * @backup: the pointer to the registers address array for backup
479 * @backup_size: the size of the registers address array for backup
480 * @realtime_en: the realtime enable register
481 * @realtime_clear: the realtime clear register
482 * @high_prio_en: the high priority enable register
483 * @high_prio_clear: the high priority clear register
484 * @interrupt_en: the interrupt enable register
485 * @interrupt_clear: the interrupt clear register
486 * @il: the pointer to struct d40_interrupt_lookup
487 * @il_size: the size of d40_interrupt_lookup array
488 * @init_reg: the pointer to the struct d40_reg_val
489 * @init_reg_size: the size of d40_reg_val array
490 */
491struct d40_gen_dmac {
492 u32 *backup;
493 u32 backup_size;
494 u32 realtime_en;
495 u32 realtime_clear;
496 u32 high_prio_en;
497 u32 high_prio_clear;
498 u32 interrupt_en;
499 u32 interrupt_clear;
500 struct d40_interrupt_lookup *il;
501 u32 il_size;
502 struct d40_reg_val *init_reg;
503 u32 init_reg_size;
504};
505
8d318a50
LW		 506/**
507 * struct d40_base - The big global struct, one for each probe'd instance.
508 *
509 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
510 * @execmd_lock: Lock for execute command usage since several channels share
511 * the same physical register.
512 * @dev: The device structure.
513 * @virtbase: The virtual base address of the DMA's register.
f4185592 514 * @rev: silicon revision detected.
8d318a50
LW		 515 * @clk: Pointer to the DMA clock structure.
516 * @phy_start: Physical memory start of the DMA registers.
517 * @phy_size: Size of the DMA register map.
518 * @irq: The IRQ number.
519 * @num_phy_chans: The number of physical channels. Read from HW. This
520 * is the number of available channels for this driver, not counting "Secure
521 * mode" allocated physical channels.
522 * @num_log_chans: The number of logical channels. Calculated from
523 * num_phy_chans.
524 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
525 * @dma_slave: dma_device channels that can do only do slave transfers.
526 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
7fb3e75e 527 * @phy_chans: Room for all possible physical channels in system.
8d318a50
LW		 528 * @log_chans: Room for all possible logical channels in system.
529 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
530 * to log_chans entries.
531 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
532 * to phy_chans entries.
533 * @plat_data: Pointer to provided platform_data which is the driver
534 * configuration.
28c7a19d 535 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
8d318a50
LW		 536 * @phy_res: Vector containing all physical channels.
537 * @lcla_pool: lcla pool settings and data.
538 * @lcpa_base: The virtual mapped address of LCPA.
539 * @phy_lcpa: The physical address of the LCPA.
540 * @lcpa_size: The size of the LCPA area.
c675b1b4 541 * @desc_slab: cache for descriptors.
7fb3e75e
N		 542 * @reg_val_backup: Here the values of some hardware registers are stored
543 * before the DMA is powered off. They are restored when the power is back on.
3cb645dc
TL		 544 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
545 * later
7fb3e75e
N		 546 * @reg_val_backup_chan: Backup data for standard channel parameter registers.
547 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
548 * @initialized: true if the dma has been initialized
3cb645dc
TL		 549 * @gen_dmac: the struct for generic registers values to represent u8500/8540
550 * DMA controller
8d318a50
LW		 551 */
552struct d40_base {
553 spinlock_t interrupt_lock;
554 spinlock_t execmd_lock;
555 struct device *dev;
556 void __iomem *virtbase;
f4185592 557 u8 rev:4;
8d318a50
LW		 558 struct clk *clk;
559 phys_addr_t phy_start;
560 resource_size_t phy_size;
561 int irq;
562 int num_phy_chans;
563 int num_log_chans;
b96710e5 564 struct device_dma_parameters dma_parms;
8d318a50
LW		 565 struct dma_device dma_both;
566 struct dma_device dma_slave;
567 struct dma_device dma_memcpy;
568 struct d40_chan *phy_chans;
569 struct d40_chan *log_chans;
570 struct d40_chan **lookup_log_chans;
571 struct d40_chan **lookup_phy_chans;
572 struct stedma40_platform_data *plat_data;
28c7a19d 573 struct regulator *lcpa_regulator;
8d318a50
LW		 574 /* Physical half channels */
575 struct d40_phy_res *phy_res;
576 struct d40_lcla_pool lcla_pool;
577 void *lcpa_base;
578 dma_addr_t phy_lcpa;
579 resource_size_t lcpa_size;
c675b1b4 580 struct kmem_cache *desc_slab;
7fb3e75e 581 u32 reg_val_backup[BACKUP_REGS_SZ];
3cb645dc 582 u32 reg_val_backup_v4[MAX(BACKUP_REGS_SZ_V4A, BACKUP_REGS_SZ_V4B)];
7fb3e75e
N		 583 u32 *reg_val_backup_chan;
584 u16 gcc_pwr_off_mask;
585 bool initialized;
3cb645dc 586 struct d40_gen_dmac gen_dmac;
8d318a50
LW		 587};
588
262d2915
RV		 589static struct device *chan2dev(struct d40_chan *d40c)
590{
591 return &d40c->chan.dev->device;
592}
593
724a8577
RV		 594static bool chan_is_physical(struct d40_chan *chan)
595{
596 return chan->log_num == D40_PHY_CHAN;
597}
598
599static bool chan_is_logical(struct d40_chan *chan)
600{
601 return !chan_is_physical(chan);
602}
603
8ca84687
RV		 604static void __iomem *chan_base(struct d40_chan *chan)
605{
606 return chan->base->virtbase + D40_DREG_PCBASE +
607 chan->phy_chan->num * D40_DREG_PCDELTA;
608}
609
6db5a8ba
RV		 610#define d40_err(dev, format, arg...) \
611 dev_err(dev, "[%s] " format, __func__, ## arg)
612
613#define chan_err(d40c, format, arg...) \
614 d40_err(chan2dev(d40c), format, ## arg)
615
b00f938c 616static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
dbd88788 617 int lli_len)
8d318a50 618{
dbd88788 619 bool is_log = chan_is_logical(d40c);
8d318a50
LW		 620 u32 align;
621 void *base;
622
623 if (is_log)
624 align = sizeof(struct d40_log_lli);
625 else
626 align = sizeof(struct d40_phy_lli);
627
628 if (lli_len == 1) {
629 base = d40d->lli_pool.pre_alloc_lli;
630 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
631 d40d->lli_pool.base = NULL;
632 } else {
594ece4d 633 d40d->lli_pool.size = lli_len * 2 * align;
8d318a50
LW		 634
635 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
636 d40d->lli_pool.base = base;
637
638 if (d40d->lli_pool.base == NULL)
639 return -ENOMEM;
640 }
641
642 if (is_log) {
d924abad 643 d40d->lli_log.src = PTR_ALIGN(base, align);
594ece4d 644 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
b00f938c
RV		 645
646 d40d->lli_pool.dma_addr = 0;
8d318a50 647 } else {
d924abad 648 d40d->lli_phy.src = PTR_ALIGN(base, align);
594ece4d 649 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
b00f938c
RV		 650
651 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
652 d40d->lli_phy.src,
653 d40d->lli_pool.size,
654 DMA_TO_DEVICE);
655
656 if (dma_mapping_error(d40c->base->dev,
657 d40d->lli_pool.dma_addr)) {
658 kfree(d40d->lli_pool.base);
659 d40d->lli_pool.base = NULL;
660 d40d->lli_pool.dma_addr = 0;
661 return -ENOMEM;
662 }
8d318a50
LW		 663 }
664
665 return 0;
666}
667
b00f938c 668static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
8d318a50 669{
b00f938c
RV		 670 if (d40d->lli_pool.dma_addr)
671 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
672 d40d->lli_pool.size, DMA_TO_DEVICE);
673
8d318a50
LW		 674 kfree(d40d->lli_pool.base);
675 d40d->lli_pool.base = NULL;
676 d40d->lli_pool.size = 0;
677 d40d->lli_log.src = NULL;
678 d40d->lli_log.dst = NULL;
679 d40d->lli_phy.src = NULL;
680 d40d->lli_phy.dst = NULL;
8d318a50
LW		 681}
682
698e4732
JA		 683static int d40_lcla_alloc_one(struct d40_chan *d40c,
684 struct d40_desc *d40d)
685{
686 unsigned long flags;
687 int i;
688 int ret = -EINVAL;
698e4732
JA		 689
690 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
691
698e4732
JA		 692 /*
693 * Allocate both src and dst at the same time, therefore the half
694 * start on 1 since 0 can't be used since zero is used as end marker.
695 */
696 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
7ce529ef
FB		 697 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
698
699 if (!d40c->base->lcla_pool.alloc_map[idx]) {
700 d40c->base->lcla_pool.alloc_map[idx] = d40d;
698e4732
JA		 701 d40d->lcla_alloc++;
702 ret = i;
703 break;
704 }
705 }
706
707 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
708
709 return ret;
710}
711
712static int d40_lcla_free_all(struct d40_chan *d40c,
713 struct d40_desc *d40d)
714{
715 unsigned long flags;
716 int i;
717 int ret = -EINVAL;
718
724a8577 719 if (chan_is_physical(d40c))
698e4732
JA		 720 return 0;
721
722 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
723
724 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
7ce529ef
FB		 725 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
726
727 if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {
728 d40c->base->lcla_pool.alloc_map[idx] = NULL;
698e4732
JA		 729 d40d->lcla_alloc--;
730 if (d40d->lcla_alloc == 0) {
731 ret = 0;
732 break;
733 }
734 }
735 }
736
737 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
738
739 return ret;
740
741}
742
8d318a50
LW		 743static void d40_desc_remove(struct d40_desc *d40d)
744{
745 list_del(&d40d->node);
746}
747
748static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
749{
a2c15fa4 750 struct d40_desc *desc = NULL;
8d318a50
LW		 751
752 if (!list_empty(&d40c->client)) {
a2c15fa4
RV		 753 struct d40_desc *d;
754 struct d40_desc *_d;
755
7fb3e75e 756 list_for_each_entry_safe(d, _d, &d40c->client, node) {
8d318a50 757 if (async_tx_test_ack(&d->txd)) {
8d318a50 758 d40_desc_remove(d);
a2c15fa4
RV		 759 desc = d;
760 memset(desc, 0, sizeof(*desc));
c675b1b4 761 break;
8d318a50 762 }
7fb3e75e 763 }
8d318a50 764 }
a2c15fa4
RV		 765
766 if (!desc)
767 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
768
769 if (desc)
770 INIT_LIST_HEAD(&desc->node);
771
772 return desc;
8d318a50
LW		 773}
774
775static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
776{
698e4732 777
b00f938c 778 d40_pool_lli_free(d40c, d40d);
698e4732 779 d40_lcla_free_all(d40c, d40d);
c675b1b4 780 kmem_cache_free(d40c->base->desc_slab, d40d);
8d318a50
LW		 781}
782
783static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
784{
785 list_add_tail(&desc->node, &d40c->active);
786}
787
1c4b0927
RV		 788static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
789{
790 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
791 struct d40_phy_lli *lli_src = desc->lli_phy.src;
792 void __iomem *base = chan_base(chan);
793
794 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
795 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
796 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
797 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
798
799 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
800 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
801 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
802 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
803}
804
4226dd86
FB		 805static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)
806{
807 list_add_tail(&desc->node, &d40c->done);
808}
809
e65889c7 810static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
698e4732 811{
e65889c7
RV		 812 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
813 struct d40_log_lli_bidir *lli = &desc->lli_log;
814 int lli_current = desc->lli_current;
815 int lli_len = desc->lli_len;
0c842b55 816 bool cyclic = desc->cyclic;
e65889c7 817 int curr_lcla = -EINVAL;
0c842b55 818 int first_lcla = 0;
28c7a19d 819 bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
0c842b55 820 bool linkback;
e65889c7 821
0c842b55
RV		 822 /*
823 * We may have partially running cyclic transfers, in case we did't get
824 * enough LCLA entries.
825 */
826 linkback = cyclic && lli_current == 0;
827
828 /*
829 * For linkback, we need one LCLA even with only one link, because we
830 * can't link back to the one in LCPA space
831 */
832 if (linkback || (lli_len - lli_current > 1)) {
7407048b
FB		 833 /*
834 * If the channel is expected to use only soft_lli don't
835 * allocate a lcla. This is to avoid a HW issue that exists
836 * in some controller during a peripheral to memory transfer
837 * that uses linked lists.
838 */
839 if (!(chan->phy_chan->use_soft_lli &&
840 chan->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM))
841 curr_lcla = d40_lcla_alloc_one(chan, desc);
842
0c842b55
RV		 843 first_lcla = curr_lcla;
844 }
845
846 /*
847 * For linkback, we normally load the LCPA in the loop since we need to
848 * link it to the second LCLA and not the first. However, if we
849 * couldn't even get a first LCLA, then we have to run in LCPA and
850 * reload manually.
851 */
852 if (!linkback || curr_lcla == -EINVAL) {
853 unsigned int flags = 0;
e65889c7 854
0c842b55
RV		 855 if (curr_lcla == -EINVAL)
856 flags |= LLI_TERM_INT;
e65889c7 857
0c842b55
RV		 858 d40_log_lli_lcpa_write(chan->lcpa,
859 &lli->dst[lli_current],
860 &lli->src[lli_current],
861 curr_lcla,
862 flags);
863 lli_current++;
864 }
6045f0bb
RV		 865
866 if (curr_lcla < 0)
867 goto out;
868
e65889c7
RV		 869 for (; lli_current < lli_len; lli_current++) {
870 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
871 8 * curr_lcla * 2;
872 struct d40_log_lli *lcla = pool->base + lcla_offset;
0c842b55 873 unsigned int flags = 0;
e65889c7
RV		 874 int next_lcla;
875
876 if (lli_current + 1 < lli_len)
877 next_lcla = d40_lcla_alloc_one(chan, desc);
878 else
0c842b55
RV		 879 next_lcla = linkback ? first_lcla : -EINVAL;
880
881 if (cyclic || next_lcla == -EINVAL)
882 flags |= LLI_TERM_INT;
e65889c7 883
0c842b55
RV		 884 if (linkback && curr_lcla == first_lcla) {
885 /* First link goes in both LCPA and LCLA */
886 d40_log_lli_lcpa_write(chan->lcpa,
887 &lli->dst[lli_current],
888 &lli->src[lli_current],
889 next_lcla, flags);
890 }
891
892 /*
893 * One unused LCLA in the cyclic case if the very first
894 * next_lcla fails...
895 */
e65889c7
RV		 896 d40_log_lli_lcla_write(lcla,
897 &lli->dst[lli_current],
898 &lli->src[lli_current],
0c842b55 899 next_lcla, flags);
e65889c7 900
28c7a19d
N		 901 /*
902 * Cache maintenance is not needed if lcla is
903 * mapped in esram
904 */
905 if (!use_esram_lcla) {
906 dma_sync_single_range_for_device(chan->base->dev,
907 pool->dma_addr, lcla_offset,
908 2 * sizeof(struct d40_log_lli),
909 DMA_TO_DEVICE);
910 }
e65889c7
RV		 911 curr_lcla = next_lcla;
912
0c842b55 913 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
e65889c7
RV		 914 lli_current++;
915 break;
916 }
917 }
918
6045f0bb 919out:
e65889c7
RV		 920 desc->lli_current = lli_current;
921}
698e4732 922
e65889c7
RV		 923static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
924{
724a8577 925 if (chan_is_physical(d40c)) {
1c4b0927 926 d40_phy_lli_load(d40c, d40d);
698e4732 927 d40d->lli_current = d40d->lli_len;
e65889c7
RV		 928 } else
929 d40_log_lli_to_lcxa(d40c, d40d);
698e4732
JA		 930}
931
8d318a50
LW		 932static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
933{
934 struct d40_desc *d;
935
936 if (list_empty(&d40c->active))
937 return NULL;
938
939 d = list_first_entry(&d40c->active,
940 struct d40_desc,
941 node);
942 return d;
943}
944
7404368c 945/* remove desc from current queue and add it to the pending_queue */
8d318a50
LW		 946static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
947{
7404368c
PF		 948 d40_desc_remove(desc);
949 desc->is_in_client_list = false;
a8f3067b
PF		 950 list_add_tail(&desc->node, &d40c->pending_queue);
951}
952
953static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
954{
955 struct d40_desc *d;
956
957 if (list_empty(&d40c->pending_queue))
958 return NULL;
959
960 d = list_first_entry(&d40c->pending_queue,
961 struct d40_desc,
962 node);
963 return d;
8d318a50
LW		 964}
965
966static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
967{
968 struct d40_desc *d;
969
970 if (list_empty(&d40c->queue))
971 return NULL;
972
973 d = list_first_entry(&d40c->queue,
974 struct d40_desc,
975 node);
976 return d;
977}
978
4226dd86
FB		 979static struct d40_desc *d40_first_done(struct d40_chan *d40c)
980{
981 if (list_empty(&d40c->done))
982 return NULL;
983
984 return list_first_entry(&d40c->done, struct d40_desc, node);
985}
986
d49278e3
PF		 987static int d40_psize_2_burst_size(bool is_log, int psize)
988{
989 if (is_log) {
990 if (psize == STEDMA40_PSIZE_LOG_1)
991 return 1;
992 } else {
993 if (psize == STEDMA40_PSIZE_PHY_1)
994 return 1;
995 }
996
997 return 2 << psize;
998}
999
1000/*
1001 * The dma only supports transmitting packages up to
1002 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
1003 * dma elements required to send the entire sg list
1004 */
1005static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
1006{
1007 int dmalen;
1008 u32 max_w = max(data_width1, data_width2);
1009 u32 min_w = min(data_width1, data_width2);
1010 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
1011
1012 if (seg_max > STEDMA40_MAX_SEG_SIZE)
1013 seg_max -= (1 << max_w);
1014
1015 if (!IS_ALIGNED(size, 1 << max_w))
1016 return -EINVAL;
1017
1018 if (size <= seg_max)
1019 dmalen = 1;
1020 else {
1021 dmalen = size / seg_max;
1022 if (dmalen * seg_max < size)
1023 dmalen++;
1024 }
1025 return dmalen;
1026}
1027
1028static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
1029 u32 data_width1, u32 data_width2)
1030{
1031 struct scatterlist *sg;
1032 int i;
1033 int len = 0;
1034 int ret;
1035
1036 for_each_sg(sgl, sg, sg_len, i) {
1037 ret = d40_size_2_dmalen(sg_dma_len(sg),
1038 data_width1, data_width2);
1039 if (ret < 0)
1040 return ret;
1041 len += ret;
1042 }
1043 return len;
1044}
8d318a50 1045
7fb3e75e
N		 1046
1047#ifdef CONFIG_PM
1048static void dma40_backup(void __iomem *baseaddr, u32 *backup,
1049 u32 *regaddr, int num, bool save)
1050{
1051 int i;
1052
1053 for (i = 0; i < num; i++) {
1054 void __iomem *addr = baseaddr + regaddr[i];
1055
1056 if (save)
1057 backup[i] = readl_relaxed(addr);
1058 else
1059 writel_relaxed(backup[i], addr);
1060 }
1061}
1062
1063static void d40_save_restore_registers(struct d40_base *base, bool save)
1064{
1065 int i;
1066
1067 /* Save/Restore channel specific registers */
1068 for (i = 0; i < base->num_phy_chans; i++) {
1069 void __iomem *addr;
1070 int idx;
1071
1072 if (base->phy_res[i].reserved)
1073 continue;
1074
1075 addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
1076 idx = i * ARRAY_SIZE(d40_backup_regs_chan);
1077
1078 dma40_backup(addr, &base->reg_val_backup_chan[idx],
1079 d40_backup_regs_chan,
1080 ARRAY_SIZE(d40_backup_regs_chan),
1081 save);
1082 }
1083
1084 /* Save/Restore global registers */
1085 dma40_backup(base->virtbase, base->reg_val_backup,
1086 d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
1087 save);
1088
1089 /* Save/Restore registers only existing on dma40 v3 and later */
3cb645dc
TL		 1090 if (base->gen_dmac.backup)
1091 dma40_backup(base->virtbase, base->reg_val_backup_v4,
1092 base->gen_dmac.backup,
1093 base->gen_dmac.backup_size,
1094 save);
7fb3e75e
N		 1095}
1096#else
1097static void d40_save_restore_registers(struct d40_base *base, bool save)
1098{
1099}
1100#endif
8d318a50 1101
1bdae6f4
N		 1102static int __d40_execute_command_phy(struct d40_chan *d40c,
1103 enum d40_command command)
8d318a50 1104{
767a9675
JA		 1105 u32 status;
1106 int i;
8d318a50
LW		 1107 void __iomem *active_reg;
1108 int ret = 0;
1109 unsigned long flags;
1d392a7b 1110 u32 wmask;
8d318a50 1111
1bdae6f4
N		 1112 if (command == D40_DMA_STOP) {
1113 ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);
1114 if (ret)
1115 return ret;
1116 }
1117
8d318a50
LW		 1118 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
1119
1120 if (d40c->phy_chan->num % 2 == 0)
1121 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1122 else
1123 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1124
1125 if (command == D40_DMA_SUSPEND_REQ) {
1126 status = (readl(active_reg) &
1127 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1128 D40_CHAN_POS(d40c->phy_chan->num);
1129
1130 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1131 goto done;
1132 }
1133
1d392a7b
JA		 1134 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
1135 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
1136 active_reg);
8d318a50
LW		 1137
1138 if (command == D40_DMA_SUSPEND_REQ) {
1139
1140 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
1141 status = (readl(active_reg) &
1142 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1143 D40_CHAN_POS(d40c->phy_chan->num);
1144
1145 cpu_relax();
1146 /*
1147 * Reduce the number of bus accesses while
1148 * waiting for the DMA to suspend.
1149 */
1150 udelay(3);
1151
1152 if (status == D40_DMA_STOP ||
1153 status == D40_DMA_SUSPENDED)
1154 break;
1155 }
1156
1157 if (i == D40_SUSPEND_MAX_IT) {
6db5a8ba
RV		 1158 chan_err(d40c,
1159 "unable to suspend the chl %d (log: %d) status %x\n",
1160 d40c->phy_chan->num, d40c->log_num,
8d318a50
LW		 1161 status);
1162 dump_stack();
1163 ret = -EBUSY;
1164 }
1165
1166 }
1167done:
1168 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
1169 return ret;
1170}
1171
1172static void d40_term_all(struct d40_chan *d40c)
1173{
1174 struct d40_desc *d40d;
7404368c 1175 struct d40_desc *_d;
8d318a50 1176
4226dd86
FB		 1177 /* Release completed descriptors */
1178 while ((d40d = d40_first_done(d40c))) {
1179 d40_desc_remove(d40d);
1180 d40_desc_free(d40c, d40d);
1181 }
1182
8d318a50
LW		 1183 /* Release active descriptors */
1184 while ((d40d = d40_first_active_get(d40c))) {
1185 d40_desc_remove(d40d);
8d318a50
LW		 1186 d40_desc_free(d40c, d40d);
1187 }
1188
1189 /* Release queued descriptors waiting for transfer */
1190 while ((d40d = d40_first_queued(d40c))) {
1191 d40_desc_remove(d40d);
8d318a50
LW		 1192 d40_desc_free(d40c, d40d);
1193 }
1194
a8f3067b
PF		 1195 /* Release pending descriptors */
1196 while ((d40d = d40_first_pending(d40c))) {
1197 d40_desc_remove(d40d);
1198 d40_desc_free(d40c, d40d);
1199 }
8d318a50 1200
7404368c
PF		 1201 /* Release client owned descriptors */
1202 if (!list_empty(&d40c->client))
1203 list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
1204 d40_desc_remove(d40d);
1205 d40_desc_free(d40c, d40d);
1206 }
1207
82babbb3
PF		 1208 /* Release descriptors in prepare queue */
1209 if (!list_empty(&d40c->prepare_queue))
1210 list_for_each_entry_safe(d40d, _d,
1211 &d40c->prepare_queue, node) {
1212 d40_desc_remove(d40d);
1213 d40_desc_free(d40c, d40d);
1214 }
7404368c 1215
8d318a50 1216 d40c->pending_tx = 0;
8d318a50
LW		 1217}
1218
1bdae6f4
N		 1219static void __d40_config_set_event(struct d40_chan *d40c,
1220 enum d40_events event_type, u32 event,
1221 int reg)
262d2915 1222{
8ca84687 1223 void __iomem *addr = chan_base(d40c) + reg;
262d2915 1224 int tries;
1bdae6f4
N		 1225 u32 status;
1226
1227 switch (event_type) {
1228
1229 case D40_DEACTIVATE_EVENTLINE:
262d2915 1230
262d2915
RV		 1231 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
1232 | ~D40_EVENTLINE_MASK(event), addr);
1bdae6f4
N		 1233 break;
1234
1235 case D40_SUSPEND_REQ_EVENTLINE:
1236 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1237 D40_EVENTLINE_POS(event);
1238
1239 if (status == D40_DEACTIVATE_EVENTLINE ||
1240 status == D40_SUSPEND_REQ_EVENTLINE)
1241 break;
262d2915 1242
1bdae6f4
N		 1243 writel((D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))
1244 | ~D40_EVENTLINE_MASK(event), addr);
1245
1246 for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
1247
1248 status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1249 D40_EVENTLINE_POS(event);
1250
1251 cpu_relax();
1252 /*
1253 * Reduce the number of bus accesses while
1254 * waiting for the DMA to suspend.
1255 */
1256 udelay(3);
1257
1258 if (status == D40_DEACTIVATE_EVENTLINE)
1259 break;
1260 }
1261
1262 if (tries == D40_SUSPEND_MAX_IT) {
1263 chan_err(d40c,
1264 "unable to stop the event_line chl %d (log: %d)"
1265 "status %x\n", d40c->phy_chan->num,
1266 d40c->log_num, status);
1267 }
1268 break;
1269
1270 case D40_ACTIVATE_EVENTLINE:
262d2915
RV		 1271 /*
1272 * The hardware sometimes doesn't register the enable when src and dst
1273 * event lines are active on the same logical channel. Retry to ensure
1274 * it does. Usually only one retry is sufficient.
1275 */
1bdae6f4
N		 1276 tries = 100;
1277 while (--tries) {
1278 writel((D40_ACTIVATE_EVENTLINE <<
1279 D40_EVENTLINE_POS(event)) |
1280 ~D40_EVENTLINE_MASK(event), addr);
262d2915 1281
1bdae6f4
N		 1282 if (readl(addr) & D40_EVENTLINE_MASK(event))
1283 break;
1284 }
262d2915 1285
1bdae6f4
N		 1286 if (tries != 99)
1287 dev_dbg(chan2dev(d40c),
1288 "[%s] workaround enable S%cLNK (%d tries)\n",
1289 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
1290 100 - tries);
262d2915 1291
1bdae6f4
N		 1292 WARN_ON(!tries);
1293 break;
262d2915 1294
1bdae6f4
N		 1295 case D40_ROUND_EVENTLINE:
1296 BUG();
1297 break;
8d318a50 1298
1bdae6f4
N		 1299 }
1300}
8d318a50 1301
1bdae6f4
N		 1302static void d40_config_set_event(struct d40_chan *d40c,
1303 enum d40_events event_type)
1304{
8d318a50
LW		 1305 /* Enable event line connected to device (or memcpy) */
1306 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1307 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
1308 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1309
1bdae6f4 1310 __d40_config_set_event(d40c, event_type, event,
262d2915 1311 D40_CHAN_REG_SSLNK);
8d318a50 1312 }
262d2915 1313
8d318a50
LW		 1314 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
1315 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1316
1bdae6f4 1317 __d40_config_set_event(d40c, event_type, event,
262d2915 1318 D40_CHAN_REG_SDLNK);
8d318a50 1319 }
8d318a50
LW		 1320}
1321
a5ebca47 1322static u32 d40_chan_has_events(struct d40_chan *d40c)
8d318a50 1323{
8ca84687 1324 void __iomem *chanbase = chan_base(d40c);
be8cb7df 1325 u32 val;
8d318a50 1326
8ca84687
RV		 1327 val = readl(chanbase + D40_CHAN_REG_SSLNK);
1328 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
be8cb7df 1329
a5ebca47 1330 return val;
8d318a50
LW		 1331}
1332
1bdae6f4
N		 1333static int
1334__d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
1335{
1336 unsigned long flags;
1337 int ret = 0;
1338 u32 active_status;
1339 void __iomem *active_reg;
1340
1341 if (d40c->phy_chan->num % 2 == 0)
1342 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1343 else
1344 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1345
1346
1347 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
1348
1349 switch (command) {
1350 case D40_DMA_STOP:
1351 case D40_DMA_SUSPEND_REQ:
1352
1353 active_status = (readl(active_reg) &
1354 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1355 D40_CHAN_POS(d40c->phy_chan->num);
1356
1357 if (active_status == D40_DMA_RUN)
1358 d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);
1359 else
1360 d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);
1361
1362 if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
1363 ret = __d40_execute_command_phy(d40c, command);
1364
1365 break;
1366
1367 case D40_DMA_RUN:
1368
1369 d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);
1370 ret = __d40_execute_command_phy(d40c, command);
1371 break;
1372
1373 case D40_DMA_SUSPENDED:
1374 BUG();
1375 break;
1376 }
1377
1378 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
1379 return ret;
1380}
1381
1382static int d40_channel_execute_command(struct d40_chan *d40c,
1383 enum d40_command command)
1384{
1385 if (chan_is_logical(d40c))
1386 return __d40_execute_command_log(d40c, command);
1387 else
1388 return __d40_execute_command_phy(d40c, command);
1389}
1390
20a5b6d0
RV		 1391static u32 d40_get_prmo(struct d40_chan *d40c)
1392{
1393 static const unsigned int phy_map[] = {
1394 [STEDMA40_PCHAN_BASIC_MODE]
1395 = D40_DREG_PRMO_PCHAN_BASIC,
1396 [STEDMA40_PCHAN_MODULO_MODE]
1397 = D40_DREG_PRMO_PCHAN_MODULO,
1398 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
1399 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
1400 };
1401 static const unsigned int log_map[] = {
1402 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
1403 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
1404 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
1405 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
1406 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
1407 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
1408 };
1409
724a8577 1410 if (chan_is_physical(d40c))
20a5b6d0
RV		 1411 return phy_map[d40c->dma_cfg.mode_opt];
1412 else
1413 return log_map[d40c->dma_cfg.mode_opt];
1414}
1415
b55912c6 1416static void d40_config_write(struct d40_chan *d40c)
8d318a50
LW		 1417{
1418 u32 addr_base;
1419 u32 var;
8d318a50
LW		 1420
1421 /* Odd addresses are even addresses + 4 */
1422 addr_base = (d40c->phy_chan->num % 2) * 4;
1423 /* Setup channel mode to logical or physical */
724a8577 1424 var = ((u32)(chan_is_logical(d40c)) + 1) <<
8d318a50
LW		 1425 D40_CHAN_POS(d40c->phy_chan->num);
1426 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
1427
1428 /* Setup operational mode option register */
20a5b6d0 1429 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
8d318a50
LW		 1430
1431 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
1432
724a8577 1433 if (chan_is_logical(d40c)) {
8ca84687
RV		 1434 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
1435 & D40_SREG_ELEM_LOG_LIDX_MASK;
1436 void __iomem *chanbase = chan_base(d40c);
1437
8d318a50 1438 /* Set default config for CFG reg */
8ca84687
RV		 1439 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
1440 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
8d318a50 1441
b55912c6 1442 /* Set LIDX for lcla */
8ca84687
RV		 1443 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
1444 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
e9f3a49c
RV		 1445
1446 /* Clear LNK which will be used by d40_chan_has_events() */
1447 writel(0, chanbase + D40_CHAN_REG_SSLNK);
1448 writel(0, chanbase + D40_CHAN_REG_SDLNK);
8d318a50 1449 }
8d318a50
LW		 1450}
1451
aa182ae2
JA		 1452static u32 d40_residue(struct d40_chan *d40c)
1453{
1454 u32 num_elt;
1455
724a8577 1456 if (chan_is_logical(d40c))
aa182ae2
JA		 1457 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1458 >> D40_MEM_LCSP2_ECNT_POS;
8ca84687
RV		 1459 else {
1460 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
1461 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
1462 >> D40_SREG_ELEM_PHY_ECNT_POS;
1463 }
1464
aa182ae2
JA		 1465 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1466}
1467
1468static bool d40_tx_is_linked(struct d40_chan *d40c)
1469{
1470 bool is_link;
1471
724a8577 1472 if (chan_is_logical(d40c))
aa182ae2
JA		 1473 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1474 else
8ca84687
RV		 1475 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
1476 & D40_SREG_LNK_PHYS_LNK_MASK;
1477
aa182ae2
JA		 1478 return is_link;
1479}
1480
86eb5fb6 1481static int d40_pause(struct d40_chan *d40c)
aa182ae2 1482{
aa182ae2
JA		 1483 int res = 0;
1484 unsigned long flags;
1485
3ac012af
JA		 1486 if (!d40c->busy)
1487 return 0;
1488
7fb3e75e 1489 pm_runtime_get_sync(d40c->base->dev);
aa182ae2
JA		 1490 spin_lock_irqsave(&d40c->lock, flags);
1491
1492 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1bdae6f4 1493
7fb3e75e
N		 1494 pm_runtime_mark_last_busy(d40c->base->dev);
1495 pm_runtime_put_autosuspend(d40c->base->dev);
aa182ae2
JA		 1496 spin_unlock_irqrestore(&d40c->lock, flags);
1497 return res;
1498}
1499
86eb5fb6 1500static int d40_resume(struct d40_chan *d40c)
aa182ae2 1501{
aa182ae2
JA		 1502 int res = 0;
1503 unsigned long flags;
1504
3ac012af
JA		 1505 if (!d40c->busy)
1506 return 0;
1507
aa182ae2 1508 spin_lock_irqsave(&d40c->lock, flags);
7fb3e75e 1509 pm_runtime_get_sync(d40c->base->dev);
aa182ae2
JA		 1510
1511 /* If bytes left to transfer or linked tx resume job */
1bdae6f4 1512 if (d40_residue(d40c) || d40_tx_is_linked(d40c))
aa182ae2 1513 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
aa182ae2 1514
7fb3e75e
N		 1515 pm_runtime_mark_last_busy(d40c->base->dev);
1516 pm_runtime_put_autosuspend(d40c->base->dev);
aa182ae2
JA		 1517 spin_unlock_irqrestore(&d40c->lock, flags);
1518 return res;
1519}
1520
8d318a50
LW		 1521static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1522{
1523 struct d40_chan *d40c = container_of(tx->chan,
1524 struct d40_chan,
1525 chan);
1526 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1527 unsigned long flags;
884485e1 1528 dma_cookie_t cookie;
8d318a50
LW		 1529
1530 spin_lock_irqsave(&d40c->lock, flags);
884485e1 1531 cookie = dma_cookie_assign(tx);
8d318a50 1532 d40_desc_queue(d40c, d40d);
8d318a50
LW		 1533 spin_unlock_irqrestore(&d40c->lock, flags);
1534
884485e1 1535 return cookie;
8d318a50
LW		 1536}
1537
1538static int d40_start(struct d40_chan *d40c)
1539{
0c32269d 1540 return d40_channel_execute_command(d40c, D40_DMA_RUN);
8d318a50
LW		 1541}
1542
1543static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1544{
1545 struct d40_desc *d40d;
1546 int err;
1547
1548 /* Start queued jobs, if any */
1549 d40d = d40_first_queued(d40c);
1550
1551 if (d40d != NULL) {
1bdae6f4 1552 if (!d40c->busy) {
7fb3e75e 1553 d40c->busy = true;
1bdae6f4
N		 1554 pm_runtime_get_sync(d40c->base->dev);
1555 }
8d318a50
LW		 1556
1557 /* Remove from queue */
1558 d40_desc_remove(d40d);
1559
1560 /* Add to active queue */
1561 d40_desc_submit(d40c, d40d);
1562
7d83a854
RV		 1563 /* Initiate DMA job */
1564 d40_desc_load(d40c, d40d);
8d318a50 1565
7d83a854
RV		 1566 /* Start dma job */
1567 err = d40_start(d40c);
8d318a50 1568
7d83a854
RV		 1569 if (err)
1570 return NULL;
8d318a50
LW		 1571 }
1572
1573 return d40d;
1574}
1575
1576/* called from interrupt context */
1577static void dma_tc_handle(struct d40_chan *d40c)
1578{
1579 struct d40_desc *d40d;
1580
8d318a50
LW		 1581 /* Get first active entry from list */
1582 d40d = d40_first_active_get(d40c);
1583
1584 if (d40d == NULL)
1585 return;
1586
0c842b55
RV		 1587 if (d40d->cyclic) {
1588 /*
1589 * If this was a paritially loaded list, we need to reloaded
1590 * it, and only when the list is completed. We need to check
1591 * for done because the interrupt will hit for every link, and
1592 * not just the last one.
1593 */
1594 if (d40d->lli_current < d40d->lli_len
1595 && !d40_tx_is_linked(d40c)
1596 && !d40_residue(d40c)) {
1597 d40_lcla_free_all(d40c, d40d);
1598 d40_desc_load(d40c, d40d);
1599 (void) d40_start(d40c);
8d318a50 1600
0c842b55
RV		 1601 if (d40d->lli_current == d40d->lli_len)
1602 d40d->lli_current = 0;
1603 }
1604 } else {
1605 d40_lcla_free_all(d40c, d40d);
8d318a50 1606
0c842b55
RV		 1607 if (d40d->lli_current < d40d->lli_len) {
1608 d40_desc_load(d40c, d40d);
1609 /* Start dma job */
1610 (void) d40_start(d40c);
1611 return;
1612 }
1613
1614 if (d40_queue_start(d40c) == NULL)
1615 d40c->busy = false;
7fb3e75e
N		 1616 pm_runtime_mark_last_busy(d40c->base->dev);
1617 pm_runtime_put_autosuspend(d40c->base->dev);
8d318a50 1618
7dd14525
FB		 1619 d40_desc_remove(d40d);
1620 d40_desc_done(d40c, d40d);
1621 }
4226dd86 1622
8d318a50
LW		 1623 d40c->pending_tx++;
1624 tasklet_schedule(&d40c->tasklet);
1625
1626}
1627
1628static void dma_tasklet(unsigned long data)
1629{
1630 struct d40_chan *d40c = (struct d40_chan *) data;
767a9675 1631 struct d40_desc *d40d;
8d318a50
LW		 1632 unsigned long flags;
1633 dma_async_tx_callback callback;
1634 void *callback_param;
1635
1636 spin_lock_irqsave(&d40c->lock, flags);
1637
4226dd86
FB		 1638 /* Get first entry from the done list */
1639 d40d = d40_first_done(d40c);
1640 if (d40d == NULL) {
1641 /* Check if we have reached here for cyclic job */
1642 d40d = d40_first_active_get(d40c);
1643 if (d40d == NULL || !d40d->cyclic)
1644 goto err;
1645 }
8d318a50 1646
0c842b55 1647 if (!d40d->cyclic)
f7fbce07 1648 dma_cookie_complete(&d40d->txd);
8d318a50
LW		 1649
1650 /*
1651 * If terminating a channel pending_tx is set to zero.
1652 * This prevents any finished active jobs to return to the client.
1653 */
1654 if (d40c->pending_tx == 0) {
1655 spin_unlock_irqrestore(&d40c->lock, flags);
1656 return;
1657 }
1658
1659 /* Callback to client */
767a9675
JA		 1660 callback = d40d->txd.callback;
1661 callback_param = d40d->txd.callback_param;
1662
0c842b55
RV		 1663 if (!d40d->cyclic) {
1664 if (async_tx_test_ack(&d40d->txd)) {
767a9675 1665 d40_desc_remove(d40d);
0c842b55 1666 d40_desc_free(d40c, d40d);
f26e03ad
FB		 1667 } else if (!d40d->is_in_client_list) {
1668 d40_desc_remove(d40d);
1669 d40_lcla_free_all(d40c, d40d);
1670 list_add_tail(&d40d->node, &d40c->client);
1671 d40d->is_in_client_list = true;
8d318a50
LW		 1672 }
1673 }
1674
1675 d40c->pending_tx--;
1676
1677 if (d40c->pending_tx)
1678 tasklet_schedule(&d40c->tasklet);
1679
1680 spin_unlock_irqrestore(&d40c->lock, flags);
1681
767a9675 1682 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
8d318a50
LW		 1683 callback(callback_param);
1684
1685 return;
1686
1bdae6f4
N		 1687err:
1688 /* Rescue manouver if receiving double interrupts */
8d318a50
LW		 1689 if (d40c->pending_tx > 0)
1690 d40c->pending_tx--;
1691 spin_unlock_irqrestore(&d40c->lock, flags);
1692}
1693
1694static irqreturn_t d40_handle_interrupt(int irq, void *data)
1695{
8d318a50 1696 int i;
8d318a50
LW		 1697 u32 idx;
1698 u32 row;
1699 long chan = -1;
1700 struct d40_chan *d40c;
1701 unsigned long flags;
1702 struct d40_base *base = data;
3cb645dc
TL		 1703 u32 regs[base->gen_dmac.il_size];
1704 struct d40_interrupt_lookup *il = base->gen_dmac.il;
1705 u32 il_size = base->gen_dmac.il_size;
8d318a50
LW		 1706
1707 spin_lock_irqsave(&base->interrupt_lock, flags);
1708
1709 /* Read interrupt status of both logical and physical channels */
3cb645dc 1710 for (i = 0; i < il_size; i++)
8d318a50
LW		 1711 regs[i] = readl(base->virtbase + il[i].src);
1712
1713 for (;;) {
1714
1715 chan = find_next_bit((unsigned long *)regs,
3cb645dc 1716 BITS_PER_LONG * il_size, chan + 1);
8d318a50
LW		 1717
1718 /* No more set bits found? */
3cb645dc 1719 if (chan == BITS_PER_LONG * il_size)
8d318a50
LW		 1720 break;
1721
1722 row = chan / BITS_PER_LONG;
1723 idx = chan & (BITS_PER_LONG - 1);
1724
8d318a50
LW		 1725 if (il[row].offset == D40_PHY_CHAN)
1726 d40c = base->lookup_phy_chans[idx];
1727 else
1728 d40c = base->lookup_log_chans[il[row].offset + idx];
53d6d68f
FB		 1729
1730 if (!d40c) {
1731 /*
1732 * No error because this can happen if something else
1733 * in the system is using the channel.
1734 */
1735 continue;
1736 }
1737
1738 /* ACK interrupt */
1739 writel(1 << idx, base->virtbase + il[row].clr);
1740
8d318a50
LW		 1741 spin_lock(&d40c->lock);
1742
1743 if (!il[row].is_error)
1744 dma_tc_handle(d40c);
1745 else
6db5a8ba
RV		 1746 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1747 chan, il[row].offset, idx);
8d318a50
LW		 1748
1749 spin_unlock(&d40c->lock);
1750 }
1751
1752 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1753
1754 return IRQ_HANDLED;
1755}
1756
8d318a50
LW		 1757static int d40_validate_conf(struct d40_chan *d40c,
1758 struct stedma40_chan_cfg *conf)
1759{
1760 int res = 0;
1761 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1762 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
38bdbf02 1763 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
8d318a50 1764
0747c7ba 1765 if (!conf->dir) {
6db5a8ba 1766 chan_err(d40c, "Invalid direction.\n");
0747c7ba
LW		 1767 res = -EINVAL;
1768 }
1769
1770 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1771 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1772 d40c->runtime_addr == 0) {
1773
6db5a8ba
RV		 1774 chan_err(d40c, "Invalid TX channel address (%d)\n",
1775 conf->dst_dev_type);
0747c7ba
LW		 1776 res = -EINVAL;
1777 }
1778
1779 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1780 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1781 d40c->runtime_addr == 0) {
6db5a8ba
RV		 1782 chan_err(d40c, "Invalid RX channel address (%d)\n",
1783 conf->src_dev_type);
0747c7ba
LW		 1784 res = -EINVAL;
1785 }
1786
1787 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
4f8fc46c 1788 conf->dst_dev_type == STEDMA40_DEV_DST_MEMORY) {
6db5a8ba 1789 chan_err(d40c, "Invalid dst\n");
8d318a50
LW		 1790 res = -EINVAL;
1791 }
1792
0747c7ba 1793 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
4f8fc46c 1794 conf->src_dev_type == STEDMA40_DEV_SRC_MEMORY) {
6db5a8ba 1795 chan_err(d40c, "Invalid src\n");
8d318a50
LW		 1796 res = -EINVAL;
1797 }
1798
4f8fc46c
LJ		 1799 if (conf->src_dev_type == STEDMA40_DEV_SRC_MEMORY &&
1800 conf->dst_dev_type == STEDMA40_DEV_DST_MEMORY && is_log) {
6db5a8ba 1801 chan_err(d40c, "No event line\n");
8d318a50
LW		 1802 res = -EINVAL;
1803 }
1804
1805 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1806 (src_event_group != dst_event_group)) {
6db5a8ba 1807 chan_err(d40c, "Invalid event group\n");
8d318a50
LW		 1808 res = -EINVAL;
1809 }
1810
1811 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1812 /*
1813 * DMAC HW supports it. Will be added to this driver,
1814 * in case any dma client requires it.
1815 */
6db5a8ba 1816 chan_err(d40c, "periph to periph not supported\n");
8d318a50
LW		 1817 res = -EINVAL;
1818 }
1819
d49278e3
PF		 1820 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1821 (1 << conf->src_info.data_width) !=
1822 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1823 (1 << conf->dst_info.data_width)) {
1824 /*
1825 * The DMAC hardware only supports
1826 * src (burst x width) == dst (burst x width)
1827 */
1828
6db5a8ba 1829 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
d49278e3
PF		 1830 res = -EINVAL;
1831 }
1832
8d318a50
LW		 1833 return res;
1834}
1835
5cd326fd
N		 1836static bool d40_alloc_mask_set(struct d40_phy_res *phy,
1837 bool is_src, int log_event_line, bool is_log,
1838 bool *first_user)
8d318a50
LW		 1839{
1840 unsigned long flags;
1841 spin_lock_irqsave(&phy->lock, flags);
5cd326fd
N		 1842
1843 *first_user = ((phy->allocated_src | phy->allocated_dst)
1844 == D40_ALLOC_FREE);
1845
4aed79b2 1846 if (!is_log) {
8d318a50
LW		 1847 /* Physical interrupts are masked per physical full channel */
1848 if (phy->allocated_src == D40_ALLOC_FREE &&
1849 phy->allocated_dst == D40_ALLOC_FREE) {
1850 phy->allocated_dst = D40_ALLOC_PHY;
1851 phy->allocated_src = D40_ALLOC_PHY;
1852 goto found;
1853 } else
1854 goto not_found;
1855 }
1856
1857 /* Logical channel */
1858 if (is_src) {
1859 if (phy->allocated_src == D40_ALLOC_PHY)
1860 goto not_found;
1861
1862 if (phy->allocated_src == D40_ALLOC_FREE)
1863 phy->allocated_src = D40_ALLOC_LOG_FREE;
1864
1865 if (!(phy->allocated_src & (1 << log_event_line))) {
1866 phy->allocated_src |= 1 << log_event_line;
1867 goto found;
1868 } else
1869 goto not_found;
1870 } else {
1871 if (phy->allocated_dst == D40_ALLOC_PHY)
1872 goto not_found;
1873
1874 if (phy->allocated_dst == D40_ALLOC_FREE)
1875 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1876
1877 if (!(phy->allocated_dst & (1 << log_event_line))) {
1878 phy->allocated_dst |= 1 << log_event_line;
1879 goto found;
1880 } else
1881 goto not_found;
1882 }
1883
1884not_found:
1885 spin_unlock_irqrestore(&phy->lock, flags);
1886 return false;
1887found:
1888 spin_unlock_irqrestore(&phy->lock, flags);
1889 return true;
1890}
1891
1892static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1893 int log_event_line)
1894{
1895 unsigned long flags;
1896 bool is_free = false;
1897
1898 spin_lock_irqsave(&phy->lock, flags);
1899 if (!log_event_line) {
8d318a50
LW		 1900 phy->allocated_dst = D40_ALLOC_FREE;
1901 phy->allocated_src = D40_ALLOC_FREE;
1902 is_free = true;
1903 goto out;
1904 }
1905
1906 /* Logical channel */
1907 if (is_src) {
1908 phy->allocated_src &= ~(1 << log_event_line);
1909 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1910 phy->allocated_src = D40_ALLOC_FREE;
1911 } else {
1912 phy->allocated_dst &= ~(1 << log_event_line);
1913 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1914 phy->allocated_dst = D40_ALLOC_FREE;
1915 }
1916
1917 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1918 D40_ALLOC_FREE);
1919
1920out:
1921 spin_unlock_irqrestore(&phy->lock, flags);
1922
1923 return is_free;
1924}
1925
5cd326fd 1926static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
8d318a50
LW		 1927{
1928 int dev_type;
1929 int event_group;
1930 int event_line;
1931 struct d40_phy_res *phys;
1932 int i;
1933 int j;
1934 int log_num;
f000df8c 1935 int num_phy_chans;
8d318a50 1936 bool is_src;
38bdbf02 1937 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
8d318a50
LW		 1938
1939 phys = d40c->base->phy_res;
f000df8c 1940 num_phy_chans = d40c->base->num_phy_chans;
8d318a50
LW		 1941
1942 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1943 dev_type = d40c->dma_cfg.src_dev_type;
1944 log_num = 2 * dev_type;
1945 is_src = true;
1946 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1947 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1948 /* dst event lines are used for logical memcpy */
1949 dev_type = d40c->dma_cfg.dst_dev_type;
1950 log_num = 2 * dev_type + 1;
1951 is_src = false;
1952 } else
1953 return -EINVAL;
1954
1955 event_group = D40_TYPE_TO_GROUP(dev_type);
1956 event_line = D40_TYPE_TO_EVENT(dev_type);
1957
1958 if (!is_log) {
1959 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1960 /* Find physical half channel */
f000df8c
GB		 1961 if (d40c->dma_cfg.use_fixed_channel) {
1962 i = d40c->dma_cfg.phy_channel;
4aed79b2 1963 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 1964 0, is_log,
1965 first_phy_user))
8d318a50 1966 goto found_phy;
f000df8c
GB		 1967 } else {
1968 for (i = 0; i < num_phy_chans; i++) {
1969 if (d40_alloc_mask_set(&phys[i], is_src,
1970 0, is_log,
1971 first_phy_user))
1972 goto found_phy;
1973 }
8d318a50
LW		 1974 }
1975 } else
1976 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1977 int phy_num = j + event_group * 2;
1978 for (i = phy_num; i < phy_num + 2; i++) {
508849ad
LW		 1979 if (d40_alloc_mask_set(&phys[i],
1980 is_src,
1981 0,
5cd326fd
N		 1982 is_log,
1983 first_phy_user))
8d318a50
LW		 1984 goto found_phy;
1985 }
1986 }
1987 return -EINVAL;
1988found_phy:
1989 d40c->phy_chan = &phys[i];
1990 d40c->log_num = D40_PHY_CHAN;
1991 goto out;
1992 }
1993 if (dev_type == -1)
1994 return -EINVAL;
1995
1996 /* Find logical channel */
1997 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1998 int phy_num = j + event_group * 2;
5cd326fd
N		 1999
2000 if (d40c->dma_cfg.use_fixed_channel) {
2001 i = d40c->dma_cfg.phy_channel;
2002
2003 if ((i != phy_num) && (i != phy_num + 1)) {
2004 dev_err(chan2dev(d40c),
2005 "invalid fixed phy channel %d\n", i);
2006 return -EINVAL;
2007 }
2008
2009 if (d40_alloc_mask_set(&phys[i], is_src, event_line,
2010 is_log, first_phy_user))
2011 goto found_log;
2012
2013 dev_err(chan2dev(d40c),
2014 "could not allocate fixed phy channel %d\n", i);
2015 return -EINVAL;
2016 }
2017
8d318a50
LW		 2018 /*
2019 * Spread logical channels across all available physical rather
2020 * than pack every logical channel at the first available phy
2021 * channels.
2022 */
2023 if (is_src) {
2024 for (i = phy_num; i < phy_num + 2; i++) {
2025 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 2026 event_line, is_log,
2027 first_phy_user))
8d318a50
LW		 2028 goto found_log;
2029 }
2030 } else {
2031 for (i = phy_num + 1; i >= phy_num; i--) {
2032 if (d40_alloc_mask_set(&phys[i], is_src,
5cd326fd
N		 2033 event_line, is_log,
2034 first_phy_user))
8d318a50
LW		 2035 goto found_log;
2036 }
2037 }
2038 }
2039 return -EINVAL;
2040
2041found_log:
2042 d40c->phy_chan = &phys[i];
2043 d40c->log_num = log_num;
2044out:
2045
2046 if (is_log)
2047 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
2048 else
2049 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
2050
2051 return 0;
2052
2053}
2054
8d318a50
LW		 2055static int d40_config_memcpy(struct d40_chan *d40c)
2056{
2057 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
2058
2059 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
29027a1e 2060 d40c->dma_cfg = dma40_memcpy_conf_log;
8d318a50 2061 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
664a57ec 2062 d40c->dma_cfg.dst_dev_type = dma40_memcpy_channels[d40c->chan.chan_id];
8d318a50
LW		 2063
2064 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
2065 dma_has_cap(DMA_SLAVE, cap)) {
29027a1e 2066 d40c->dma_cfg = dma40_memcpy_conf_phy;
8d318a50 2067 } else {
6db5a8ba 2068 chan_err(d40c, "No memcpy\n");
8d318a50
LW		 2069 return -EINVAL;
2070 }
2071
2072 return 0;
2073}
2074
8d318a50
LW		 2075static int d40_free_dma(struct d40_chan *d40c)
2076{
2077
2078 int res = 0;
d181b3a8 2079 u32 event;
8d318a50
LW		 2080 struct d40_phy_res *phy = d40c->phy_chan;
2081 bool is_src;
2082
2083 /* Terminate all queued and active transfers */
2084 d40_term_all(d40c);
2085
2086 if (phy == NULL) {
6db5a8ba 2087 chan_err(d40c, "phy == null\n");
8d318a50
LW		 2088 return -EINVAL;
2089 }
2090
2091 if (phy->allocated_src == D40_ALLOC_FREE &&
2092 phy->allocated_dst == D40_ALLOC_FREE) {
6db5a8ba 2093 chan_err(d40c, "channel already free\n");
8d318a50
LW		 2094 return -EINVAL;
2095 }
2096
8d318a50
LW		 2097 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
2098 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
2099 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
8d318a50
LW		 2100 is_src = false;
2101 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
2102 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
8d318a50
LW		 2103 is_src = true;
2104 } else {
6db5a8ba 2105 chan_err(d40c, "Unknown direction\n");
8d318a50
LW		 2106 return -EINVAL;
2107 }
2108
7fb3e75e 2109 pm_runtime_get_sync(d40c->base->dev);
1bdae6f4 2110 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
d181b3a8 2111 if (res) {
1bdae6f4 2112 chan_err(d40c, "stop failed\n");
7fb3e75e 2113 goto out;
d181b3a8
JA		 2114 }
2115
1bdae6f4 2116 d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
8d318a50 2117
1bdae6f4 2118 if (chan_is_logical(d40c))
8d318a50 2119 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1bdae6f4
N		 2120 else
2121 d40c->base->lookup_phy_chans[phy->num] = NULL;
7fb3e75e
N		 2122
2123 if (d40c->busy) {
2124 pm_runtime_mark_last_busy(d40c->base->dev);
2125 pm_runtime_put_autosuspend(d40c->base->dev);
2126 }
2127
2128 d40c->busy = false;
8d318a50 2129 d40c->phy_chan = NULL;
ce2ca125 2130 d40c->configured = false;
7fb3e75e 2131out:
8d318a50 2132
7fb3e75e
N		 2133 pm_runtime_mark_last_busy(d40c->base->dev);
2134 pm_runtime_put_autosuspend(d40c->base->dev);
2135 return res;
8d318a50
LW		 2136}
2137
a5ebca47
JA		 2138static bool d40_is_paused(struct d40_chan *d40c)
2139{
8ca84687 2140 void __iomem *chanbase = chan_base(d40c);
a5ebca47
JA		 2141 bool is_paused = false;
2142 unsigned long flags;
2143 void __iomem *active_reg;
2144 u32 status;
2145 u32 event;
a5ebca47
JA		 2146
2147 spin_lock_irqsave(&d40c->lock, flags);
2148
724a8577 2149 if (chan_is_physical(d40c)) {
a5ebca47
JA		 2150 if (d40c->phy_chan->num % 2 == 0)
2151 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
2152 else
2153 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
2154
2155 status = (readl(active_reg) &
2156 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
2157 D40_CHAN_POS(d40c->phy_chan->num);
2158 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
2159 is_paused = true;
2160
2161 goto _exit;
2162 }
2163
a5ebca47 2164 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
9dbfbd35 2165 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
a5ebca47 2166 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
8ca84687 2167 status = readl(chanbase + D40_CHAN_REG_SDLNK);
9dbfbd35 2168 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
a5ebca47 2169 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
8ca84687 2170 status = readl(chanbase + D40_CHAN_REG_SSLNK);
9dbfbd35 2171 } else {
6db5a8ba 2172 chan_err(d40c, "Unknown direction\n");
a5ebca47
JA		 2173 goto _exit;
2174 }
9dbfbd35 2175
a5ebca47
JA		 2176 status = (status & D40_EVENTLINE_MASK(event)) >>
2177 D40_EVENTLINE_POS(event);
2178
2179 if (status != D40_DMA_RUN)
2180 is_paused = true;
a5ebca47
JA		 2181_exit:
2182 spin_unlock_irqrestore(&d40c->lock, flags);
2183 return is_paused;
2184
2185}
2186
8d318a50
LW		 2187static u32 stedma40_residue(struct dma_chan *chan)
2188{
2189 struct d40_chan *d40c =
2190 container_of(chan, struct d40_chan, chan);
2191 u32 bytes_left;
2192 unsigned long flags;
2193
2194 spin_lock_irqsave(&d40c->lock, flags);
2195 bytes_left = d40_residue(d40c);
2196 spin_unlock_irqrestore(&d40c->lock, flags);
2197
2198 return bytes_left;
2199}
2200
3e3a0763
RV		 2201static int
2202d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
2203 struct scatterlist *sg_src, struct scatterlist *sg_dst,
822c5676
RV		 2204 unsigned int sg_len, dma_addr_t src_dev_addr,
2205 dma_addr_t dst_dev_addr)
3e3a0763
RV		 2206{
2207 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2208 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2209 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
5ed04b85 2210 int ret;
3e3a0763 2211
5ed04b85
RV		 2212 ret = d40_log_sg_to_lli(sg_src, sg_len,
2213 src_dev_addr,
2214 desc->lli_log.src,
2215 chan->log_def.lcsp1,
2216 src_info->data_width,
2217 dst_info->data_width);
2218
2219 ret = d40_log_sg_to_lli(sg_dst, sg_len,
2220 dst_dev_addr,
2221 desc->lli_log.dst,
2222 chan->log_def.lcsp3,
2223 dst_info->data_width,
2224 src_info->data_width);
2225
2226 return ret < 0 ? ret : 0;
3e3a0763
RV		 2227}
2228
2229static int
2230d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
2231 struct scatterlist *sg_src, struct scatterlist *sg_dst,
822c5676
RV		 2232 unsigned int sg_len, dma_addr_t src_dev_addr,
2233 dma_addr_t dst_dev_addr)
3e3a0763 2234{
3e3a0763
RV		 2235 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2236 struct stedma40_half_channel_info *src_info = &cfg->src_info;
2237 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
0c842b55 2238 unsigned long flags = 0;
3e3a0763
RV		 2239 int ret;
2240
0c842b55
RV		 2241 if (desc->cyclic)
2242 flags |= LLI_CYCLIC | LLI_TERM_INT;
2243
3e3a0763
RV		 2244 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
2245 desc->lli_phy.src,
2246 virt_to_phys(desc->lli_phy.src),
2247 chan->src_def_cfg,
0c842b55 2248 src_info, dst_info, flags);
3e3a0763
RV		 2249
2250 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
2251 desc->lli_phy.dst,
2252 virt_to_phys(desc->lli_phy.dst),
2253 chan->dst_def_cfg,
0c842b55 2254 dst_info, src_info, flags);
3e3a0763
RV		 2255
2256 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
2257 desc->lli_pool.size, DMA_TO_DEVICE);
2258
2259 return ret < 0 ? ret : 0;
2260}
2261
5f81158f
RV		 2262static struct d40_desc *
2263d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
2264 unsigned int sg_len, unsigned long dma_flags)
2265{
2266 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2267 struct d40_desc *desc;
dbd88788 2268 int ret;
5f81158f
RV		 2269
2270 desc = d40_desc_get(chan);
2271 if (!desc)
2272 return NULL;
2273
2274 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
2275 cfg->dst_info.data_width);
2276 if (desc->lli_len < 0) {
2277 chan_err(chan, "Unaligned size\n");
dbd88788
RV		 2278 goto err;
2279 }
5f81158f 2280
dbd88788
RV		 2281 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
2282 if (ret < 0) {
2283 chan_err(chan, "Could not allocate lli\n");
2284 goto err;
5f81158f
RV		 2285 }
2286
2287 desc->lli_current = 0;
2288 desc->txd.flags = dma_flags;
2289 desc->txd.tx_submit = d40_tx_submit;
2290
2291 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
2292
2293 return desc;
dbd88788
RV		 2294
2295err:
2296 d40_desc_free(chan, desc);
2297 return NULL;
5f81158f
RV		 2298}
2299
cade1d30 2300static dma_addr_t
db8196df 2301d40_get_dev_addr(struct d40_chan *chan, enum dma_transfer_direction direction)
8d318a50 2302{
cade1d30
RV		 2303 struct stedma40_platform_data *plat = chan->base->plat_data;
2304 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
711b9cea 2305 dma_addr_t addr = 0;
cade1d30
RV		 2306
2307 if (chan->runtime_addr)
2308 return chan->runtime_addr;
2309
db8196df 2310 if (direction == DMA_DEV_TO_MEM)
cade1d30 2311 addr = plat->dev_rx[cfg->src_dev_type];
db8196df 2312 else if (direction == DMA_MEM_TO_DEV)
cade1d30
RV		 2313 addr = plat->dev_tx[cfg->dst_dev_type];
2314
2315 return addr;
2316}
2317
2318static struct dma_async_tx_descriptor *
2319d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
2320 struct scatterlist *sg_dst, unsigned int sg_len,
db8196df 2321 enum dma_transfer_direction direction, unsigned long dma_flags)
cade1d30
RV		 2322{
2323 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
822c5676
RV		 2324 dma_addr_t src_dev_addr = 0;
2325 dma_addr_t dst_dev_addr = 0;
cade1d30 2326 struct d40_desc *desc;
2a614340 2327 unsigned long flags;
cade1d30 2328 int ret;
8d318a50 2329
cade1d30
RV		 2330 if (!chan->phy_chan) {
2331 chan_err(chan, "Cannot prepare unallocated channel\n");
2332 return NULL;
0d0f6b8b
JA		 2333 }
2334
cade1d30 2335 spin_lock_irqsave(&chan->lock, flags);
8d318a50 2336
cade1d30
RV		 2337 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
2338 if (desc == NULL)
8d318a50
LW		 2339 goto err;
2340
0c842b55
RV		 2341 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
2342 desc->cyclic = true;
2343
7e426da8 2344 if (direction != DMA_TRANS_NONE) {
822c5676
RV		 2345 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
2346
db8196df 2347 if (direction == DMA_DEV_TO_MEM)
822c5676 2348 src_dev_addr = dev_addr;
db8196df 2349 else if (direction == DMA_MEM_TO_DEV)
822c5676
RV		 2350 dst_dev_addr = dev_addr;
2351 }
cade1d30
RV		 2352
2353 if (chan_is_logical(chan))
2354 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
822c5676 2355 sg_len, src_dev_addr, dst_dev_addr);
cade1d30
RV		 2356 else
2357 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
822c5676 2358 sg_len, src_dev_addr, dst_dev_addr);
cade1d30
RV		 2359
2360 if (ret) {
2361 chan_err(chan, "Failed to prepare %s sg job: %d\n",
2362 chan_is_logical(chan) ? "log" : "phy", ret);
2363 goto err;
8d318a50
LW		 2364 }
2365
82babbb3
PF		 2366 /*
2367 * add descriptor to the prepare queue in order to be able
2368 * to free them later in terminate_all
2369 */
2370 list_add_tail(&desc->node, &chan->prepare_queue);
2371
cade1d30
RV		 2372 spin_unlock_irqrestore(&chan->lock, flags);
2373
2374 return &desc->txd;
8d318a50 2375
8d318a50 2376err:
cade1d30
RV		 2377 if (desc)
2378 d40_desc_free(chan, desc);
2379 spin_unlock_irqrestore(&chan->lock, flags);
8d318a50
LW		 2380 return NULL;
2381}
8d318a50
LW		 2382
2383bool stedma40_filter(struct dma_chan *chan, void *data)
2384{
2385 struct stedma40_chan_cfg *info = data;
2386 struct d40_chan *d40c =
2387 container_of(chan, struct d40_chan, chan);
2388 int err;
2389
2390 if (data) {
2391 err = d40_validate_conf(d40c, info);
2392 if (!err)
2393 d40c->dma_cfg = *info;
2394 } else
2395 err = d40_config_memcpy(d40c);
2396
ce2ca125
RV		 2397 if (!err)
2398 d40c->configured = true;
2399
8d318a50
LW		 2400 return err == 0;
2401}
2402EXPORT_SYMBOL(stedma40_filter);
2403
ac2c0a38
RV		 2404static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
2405{
2406 bool realtime = d40c->dma_cfg.realtime;
2407 bool highprio = d40c->dma_cfg.high_priority;
3cb645dc 2408 u32 rtreg;
ac2c0a38
RV		 2409 u32 event = D40_TYPE_TO_EVENT(dev_type);
2410 u32 group = D40_TYPE_TO_GROUP(dev_type);
2411 u32 bit = 1 << event;
ccc3d697 2412 u32 prioreg;
3cb645dc 2413 struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;
ccc3d697 2414
3cb645dc 2415 rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;
ccc3d697
RV		 2416 /*
2417 * Due to a hardware bug, in some cases a logical channel triggered by
2418 * a high priority destination event line can generate extra packet
2419 * transactions.
2420 *
2421 * The workaround is to not set the high priority level for the
2422 * destination event lines that trigger logical channels.
2423 */
2424 if (!src && chan_is_logical(d40c))
2425 highprio = false;
2426
3cb645dc 2427 prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;
ac2c0a38
RV		 2428
2429 /* Destination event lines are stored in the upper halfword */
2430 if (!src)
2431 bit <<= 16;
2432
2433 writel(bit, d40c->base->virtbase + prioreg + group * 4);
2434 writel(bit, d40c->base->virtbase + rtreg + group * 4);
2435}
2436
2437static void d40_set_prio_realtime(struct d40_chan *d40c)
2438{
2439 if (d40c->base->rev < 3)
2440 return;
2441
2442 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
2443 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2444 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
2445
2446 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
2447 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
2448 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
2449}
2450
8d318a50
LW		 2451/* DMA ENGINE functions */
2452static int d40_alloc_chan_resources(struct dma_chan *chan)
2453{
2454 int err;
2455 unsigned long flags;
2456 struct d40_chan *d40c =
2457 container_of(chan, struct d40_chan, chan);
ef1872ec 2458 bool is_free_phy;
8d318a50
LW		 2459 spin_lock_irqsave(&d40c->lock, flags);
2460
d3ee98cd 2461 dma_cookie_init(chan);
8d318a50 2462
ce2ca125
RV		 2463 /* If no dma configuration is set use default configuration (memcpy) */
2464 if (!d40c->configured) {
8d318a50 2465 err = d40_config_memcpy(d40c);
ff0b12ba 2466 if (err) {
6db5a8ba 2467 chan_err(d40c, "Failed to configure memcpy channel\n");
ff0b12ba
JA		 2468 goto fail;
2469 }
8d318a50
LW		 2470 }
2471
5cd326fd 2472 err = d40_allocate_channel(d40c, &is_free_phy);
8d318a50 2473 if (err) {
6db5a8ba 2474 chan_err(d40c, "Failed to allocate channel\n");
7fb3e75e 2475 d40c->configured = false;
ff0b12ba 2476 goto fail;
8d318a50
LW		 2477 }
2478
7fb3e75e 2479 pm_runtime_get_sync(d40c->base->dev);
ef1872ec
LW		 2480 /* Fill in basic CFG register values */
2481 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
724a8577 2482 &d40c->dst_def_cfg, chan_is_logical(d40c));
ef1872ec 2483
ac2c0a38
RV		 2484 d40_set_prio_realtime(d40c);
2485
724a8577 2486 if (chan_is_logical(d40c)) {
ef1872ec
LW		 2487 d40_log_cfg(&d40c->dma_cfg,
2488 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2489
2490 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
2491 d40c->lcpa = d40c->base->lcpa_base +
f26e03ad 2492 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
ef1872ec
LW		 2493 else
2494 d40c->lcpa = d40c->base->lcpa_base +
f26e03ad
FB		 2495 d40c->dma_cfg.dst_dev_type *
2496 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
ef1872ec
LW		 2497 }
2498
5cd326fd
N		 2499 dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
2500 chan_is_logical(d40c) ? "logical" : "physical",
2501 d40c->phy_chan->num,
2502 d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
2503
2504
ef1872ec
LW		 2505 /*
2506 * Only write channel configuration to the DMA if the physical
2507 * resource is free. In case of multiple logical channels
2508 * on the same physical resource, only the first write is necessary.
2509 */
b55912c6
JA		 2510 if (is_free_phy)
2511 d40_config_write(d40c);
ff0b12ba 2512fail:
7fb3e75e
N		 2513 pm_runtime_mark_last_busy(d40c->base->dev);
2514 pm_runtime_put_autosuspend(d40c->base->dev);
8d318a50 2515 spin_unlock_irqrestore(&d40c->lock, flags);
ff0b12ba 2516 return err;
8d318a50
LW		 2517}
2518
2519static void d40_free_chan_resources(struct dma_chan *chan)
2520{
2521 struct d40_chan *d40c =
2522 container_of(chan, struct d40_chan, chan);
2523 int err;
2524 unsigned long flags;
2525
0d0f6b8b 2526 if (d40c->phy_chan == NULL) {
6db5a8ba 2527 chan_err(d40c, "Cannot free unallocated channel\n");
0d0f6b8b
JA		 2528 return;
2529 }
2530
8d318a50
LW		 2531 spin_lock_irqsave(&d40c->lock, flags);
2532
2533 err = d40_free_dma(d40c);
2534
2535 if (err)
6db5a8ba 2536 chan_err(d40c, "Failed to free channel\n");
8d318a50
LW		 2537 spin_unlock_irqrestore(&d40c->lock, flags);
2538}
2539
2540static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2541 dma_addr_t dst,
2542 dma_addr_t src,
2543 size_t size,
2a614340 2544 unsigned long dma_flags)
8d318a50 2545{
95944c6e
RV		 2546 struct scatterlist dst_sg;
2547 struct scatterlist src_sg;
8d318a50 2548
95944c6e
RV		 2549 sg_init_table(&dst_sg, 1);
2550 sg_init_table(&src_sg, 1);
8d318a50 2551
95944c6e
RV		 2552 sg_dma_address(&dst_sg) = dst;
2553 sg_dma_address(&src_sg) = src;
8d318a50 2554
95944c6e
RV		 2555 sg_dma_len(&dst_sg) = size;
2556 sg_dma_len(&src_sg) = size;
8d318a50 2557
cade1d30 2558 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
8d318a50
LW		 2559}
2560
0d688662 2561static struct dma_async_tx_descriptor *
cade1d30
RV		 2562d40_prep_memcpy_sg(struct dma_chan *chan,
2563 struct scatterlist *dst_sg, unsigned int dst_nents,
2564 struct scatterlist *src_sg, unsigned int src_nents,
2565 unsigned long dma_flags)
0d688662
IS		 2566{
2567 if (dst_nents != src_nents)
2568 return NULL;
2569
cade1d30 2570 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
00ac0341
RV		 2571}
2572
f26e03ad
FB		 2573static struct dma_async_tx_descriptor *
2574d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2575 unsigned int sg_len, enum dma_transfer_direction direction,
2576 unsigned long dma_flags, void *context)
8d318a50 2577{
a725dcc0 2578 if (!is_slave_direction(direction))
00ac0341
RV		 2579 return NULL;
2580
cade1d30 2581 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
8d318a50
LW		 2582}
2583
0c842b55
RV		 2584static struct dma_async_tx_descriptor *
2585dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2586 size_t buf_len, size_t period_len,
ec8b5e48
PU		 2587 enum dma_transfer_direction direction, unsigned long flags,
2588 void *context)
0c842b55
RV		 2589{
2590 unsigned int periods = buf_len / period_len;
2591 struct dma_async_tx_descriptor *txd;
2592 struct scatterlist *sg;
2593 int i;
2594
79ca7ec3 2595 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);
0c842b55
RV		 2596 for (i = 0; i < periods; i++) {
2597 sg_dma_address(&sg[i]) = dma_addr;
2598 sg_dma_len(&sg[i]) = period_len;
2599 dma_addr += period_len;
2600 }
2601
2602 sg[periods].offset = 0;
fdaf9c4b 2603 sg_dma_len(&sg[periods]) = 0;
0c842b55
RV		 2604 sg[periods].page_link =
2605 ((unsigned long)sg | 0x01) & ~0x02;
2606
2607 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2608 DMA_PREP_INTERRUPT);
2609
2610 kfree(sg);
2611
2612 return txd;
2613}
2614
8d318a50
LW		 2615static enum dma_status d40_tx_status(struct dma_chan *chan,
2616 dma_cookie_t cookie,
2617 struct dma_tx_state *txstate)
2618{
2619 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
96a2af41 2620 enum dma_status ret;
8d318a50 2621
0d0f6b8b 2622 if (d40c->phy_chan == NULL) {
6db5a8ba 2623 chan_err(d40c, "Cannot read status of unallocated channel\n");
0d0f6b8b
JA		 2624 return -EINVAL;
2625 }
2626
96a2af41
RKAL		 2627 ret = dma_cookie_status(chan, cookie, txstate);
2628 if (ret != DMA_SUCCESS)
2629 dma_set_residue(txstate, stedma40_residue(chan));
8d318a50 2630
a5ebca47
JA		 2631 if (d40_is_paused(d40c))
2632 ret = DMA_PAUSED;
8d318a50
LW		 2633
2634 return ret;
2635}
2636
2637static void d40_issue_pending(struct dma_chan *chan)
2638{
2639 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2640 unsigned long flags;
2641
0d0f6b8b 2642 if (d40c->phy_chan == NULL) {
6db5a8ba 2643 chan_err(d40c, "Channel is not allocated!\n");
0d0f6b8b
JA		 2644 return;
2645 }
2646
8d318a50
LW		 2647 spin_lock_irqsave(&d40c->lock, flags);
2648
a8f3067b
PF		 2649 list_splice_tail_init(&d40c->pending_queue, &d40c->queue);
2650
2651 /* Busy means that queued jobs are already being processed */
8d318a50
LW		 2652 if (!d40c->busy)
2653 (void) d40_queue_start(d40c);
2654
2655 spin_unlock_irqrestore(&d40c->lock, flags);
2656}
2657
1bdae6f4
N		 2658static void d40_terminate_all(struct dma_chan *chan)
2659{
2660 unsigned long flags;
2661 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2662 int ret;
2663
2664 spin_lock_irqsave(&d40c->lock, flags);
2665
2666 pm_runtime_get_sync(d40c->base->dev);
2667 ret = d40_channel_execute_command(d40c, D40_DMA_STOP);
2668 if (ret)
2669 chan_err(d40c, "Failed to stop channel\n");
2670
2671 d40_term_all(d40c);
2672 pm_runtime_mark_last_busy(d40c->base->dev);
2673 pm_runtime_put_autosuspend(d40c->base->dev);
2674 if (d40c->busy) {
2675 pm_runtime_mark_last_busy(d40c->base->dev);
2676 pm_runtime_put_autosuspend(d40c->base->dev);
2677 }
2678 d40c->busy = false;
2679
2680 spin_unlock_irqrestore(&d40c->lock, flags);
2681}
2682
98ca5289
RV		 2683static int
2684dma40_config_to_halfchannel(struct d40_chan *d40c,
2685 struct stedma40_half_channel_info *info,
2686 enum dma_slave_buswidth width,
2687 u32 maxburst)
2688{
2689 enum stedma40_periph_data_width addr_width;
2690 int psize;
2691
2692 switch (width) {
2693 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2694 addr_width = STEDMA40_BYTE_WIDTH;
2695 break;
2696 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2697 addr_width = STEDMA40_HALFWORD_WIDTH;
2698 break;
2699 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2700 addr_width = STEDMA40_WORD_WIDTH;
2701 break;
2702 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2703 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2704 break;
2705 default:
2706 dev_err(d40c->base->dev,
2707 "illegal peripheral address width "
2708 "requested (%d)\n",
2709 width);
2710 return -EINVAL;
2711 }
2712
2713 if (chan_is_logical(d40c)) {
2714 if (maxburst >= 16)
2715 psize = STEDMA40_PSIZE_LOG_16;
2716 else if (maxburst >= 8)
2717 psize = STEDMA40_PSIZE_LOG_8;
2718 else if (maxburst >= 4)
2719 psize = STEDMA40_PSIZE_LOG_4;
2720 else
2721 psize = STEDMA40_PSIZE_LOG_1;
2722 } else {
2723 if (maxburst >= 16)
2724 psize = STEDMA40_PSIZE_PHY_16;
2725 else if (maxburst >= 8)
2726 psize = STEDMA40_PSIZE_PHY_8;
2727 else if (maxburst >= 4)
2728 psize = STEDMA40_PSIZE_PHY_4;
2729 else
2730 psize = STEDMA40_PSIZE_PHY_1;
2731 }
2732
2733 info->data_width = addr_width;
2734 info->psize = psize;
2735 info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2736
2737 return 0;
2738}
2739
95e1400f 2740/* Runtime reconfiguration extension */
98ca5289
RV		 2741static int d40_set_runtime_config(struct dma_chan *chan,
2742 struct dma_slave_config *config)
95e1400f
LW		 2743{
2744 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2745 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
98ca5289 2746 enum dma_slave_buswidth src_addr_width, dst_addr_width;
95e1400f 2747 dma_addr_t config_addr;
98ca5289
RV		 2748 u32 src_maxburst, dst_maxburst;
2749 int ret;
2750
2751 src_addr_width = config->src_addr_width;
2752 src_maxburst = config->src_maxburst;
2753 dst_addr_width = config->dst_addr_width;
2754 dst_maxburst = config->dst_maxburst;
95e1400f 2755
db8196df 2756 if (config->direction == DMA_DEV_TO_MEM) {
95e1400f
LW		 2757 dma_addr_t dev_addr_rx =
2758 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2759
2760 config_addr = config->src_addr;
2761 if (dev_addr_rx)
2762 dev_dbg(d40c->base->dev,
2763 "channel has a pre-wired RX address %08x "
2764 "overriding with %08x\n",
2765 dev_addr_rx, config_addr);
2766 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2767 dev_dbg(d40c->base->dev,
2768 "channel was not configured for peripheral "
2769 "to memory transfer (%d) overriding\n",
2770 cfg->dir);
2771 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2772
98ca5289
RV		 2773 /* Configure the memory side */
2774 if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2775 dst_addr_width = src_addr_width;
2776 if (dst_maxburst == 0)
2777 dst_maxburst = src_maxburst;
95e1400f 2778
db8196df 2779 } else if (config->direction == DMA_MEM_TO_DEV) {
95e1400f
LW		 2780 dma_addr_t dev_addr_tx =
2781 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2782
2783 config_addr = config->dst_addr;
2784 if (dev_addr_tx)
2785 dev_dbg(d40c->base->dev,
2786 "channel has a pre-wired TX address %08x "
2787 "overriding with %08x\n",
2788 dev_addr_tx, config_addr);
2789 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2790 dev_dbg(d40c->base->dev,
2791 "channel was not configured for memory "
2792 "to peripheral transfer (%d) overriding\n",
2793 cfg->dir);
2794 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2795
98ca5289
RV		 2796 /* Configure the memory side */
2797 if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2798 src_addr_width = dst_addr_width;
2799 if (src_maxburst == 0)
2800 src_maxburst = dst_maxburst;
95e1400f
LW		 2801 } else {
2802 dev_err(d40c->base->dev,
2803 "unrecognized channel direction %d\n",
2804 config->direction);
98ca5289 2805 return -EINVAL;
95e1400f
LW		 2806 }
2807
98ca5289 2808 if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
95e1400f 2809 dev_err(d40c->base->dev,
98ca5289
RV		 2810 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2811 src_maxburst,
2812 src_addr_width,
2813 dst_maxburst,
2814 dst_addr_width);
2815 return -EINVAL;
95e1400f
LW		 2816 }
2817
92bb6cdb
PF		 2818 if (src_maxburst > 16) {
2819 src_maxburst = 16;
2820 dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;
2821 } else if (dst_maxburst > 16) {
2822 dst_maxburst = 16;
2823 src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;
2824 }
2825
98ca5289
RV		 2826 ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,
2827 src_addr_width,
2828 src_maxburst);
2829 if (ret)
2830 return ret;
95e1400f 2831
98ca5289
RV		 2832 ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,
2833 dst_addr_width,
2834 dst_maxburst);
2835 if (ret)
2836 return ret;
95e1400f 2837
a59670a4 2838 /* Fill in register values */
724a8577 2839 if (chan_is_logical(d40c))
a59670a4
PF		 2840 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2841 else
2842 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2843 &d40c->dst_def_cfg, false);
2844
95e1400f
LW		 2845 /* These settings will take precedence later */
2846 d40c->runtime_addr = config_addr;
2847 d40c->runtime_direction = config->direction;
2848 dev_dbg(d40c->base->dev,
98ca5289
RV		 2849 "configured channel %s for %s, data width %d/%d, "
2850 "maxburst %d/%d elements, LE, no flow control\n",
95e1400f 2851 dma_chan_name(chan),
db8196df 2852 (config->direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
98ca5289
RV		 2853 src_addr_width, dst_addr_width,
2854 src_maxburst, dst_maxburst);
2855
2856 return 0;
95e1400f
LW		 2857}
2858
05827630
LW		 2859static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2860 unsigned long arg)
8d318a50 2861{
8d318a50
LW		 2862 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2863
0d0f6b8b 2864 if (d40c->phy_chan == NULL) {
6db5a8ba 2865 chan_err(d40c, "Channel is not allocated!\n");
0d0f6b8b
JA		 2866 return -EINVAL;
2867 }
2868
8d318a50
LW		 2869 switch (cmd) {
2870 case DMA_TERMINATE_ALL:
1bdae6f4
N		 2871 d40_terminate_all(chan);
2872 return 0;
8d318a50 2873 case DMA_PAUSE:
86eb5fb6 2874 return d40_pause(d40c);
8d318a50 2875 case DMA_RESUME:
86eb5fb6 2876 return d40_resume(d40c);
95e1400f 2877 case DMA_SLAVE_CONFIG:
98ca5289 2878 return d40_set_runtime_config(chan,
95e1400f 2879 (struct dma_slave_config *) arg);
95e1400f
LW		 2880 default:
2881 break;
8d318a50
LW		 2882 }
2883
2884 /* Other commands are unimplemented */
2885 return -ENXIO;
2886}
2887
2888/* Initialization functions */
2889
2890static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2891 struct d40_chan *chans, int offset,
2892 int num_chans)
2893{
2894 int i = 0;
2895 struct d40_chan *d40c;
2896
2897 INIT_LIST_HEAD(&dma->channels);
2898
2899 for (i = offset; i < offset + num_chans; i++) {
2900 d40c = &chans[i];
2901 d40c->base = base;
2902 d40c->chan.device = dma;
2903
8d318a50
LW		 2904 spin_lock_init(&d40c->lock);
2905
2906 d40c->log_num = D40_PHY_CHAN;
2907
4226dd86 2908 INIT_LIST_HEAD(&d40c->done);
8d318a50
LW		 2909 INIT_LIST_HEAD(&d40c->active);
2910 INIT_LIST_HEAD(&d40c->queue);
a8f3067b 2911 INIT_LIST_HEAD(&d40c->pending_queue);
8d318a50 2912 INIT_LIST_HEAD(&d40c->client);
82babbb3 2913 INIT_LIST_HEAD(&d40c->prepare_queue);
8d318a50 2914
8d318a50
LW		 2915 tasklet_init(&d40c->tasklet, dma_tasklet,
2916 (unsigned long) d40c);
2917
2918 list_add_tail(&d40c->chan.device_node,
2919 &dma->channels);
2920 }
2921}
2922
7ad74a7c
RV		 2923static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2924{
2925 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2926 dev->device_prep_slave_sg = d40_prep_slave_sg;
2927
2928 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2929 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2930
2931 /*
2932 * This controller can only access address at even
2933 * 32bit boundaries, i.e. 2^2
2934 */
2935 dev->copy_align = 2;
2936 }
2937
2938 if (dma_has_cap(DMA_SG, dev->cap_mask))
2939 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2940
0c842b55
RV		 2941 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2942 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2943
7ad74a7c
RV		 2944 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2945 dev->device_free_chan_resources = d40_free_chan_resources;
2946 dev->device_issue_pending = d40_issue_pending;
2947 dev->device_tx_status = d40_tx_status;
2948 dev->device_control = d40_control;
2949 dev->dev = base->dev;
2950}
2951
8d318a50
LW		 2952static int __init d40_dmaengine_init(struct d40_base *base,
2953 int num_reserved_chans)
2954{
2955 int err ;
2956
2957 d40_chan_init(base, &base->dma_slave, base->log_chans,
2958 0, base->num_log_chans);
2959
2960 dma_cap_zero(base->dma_slave.cap_mask);
2961 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
0c842b55 2962 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
8d318a50 2963
7ad74a7c 2964 d40_ops_init(base, &base->dma_slave);
8d318a50
LW		 2965
2966 err = dma_async_device_register(&base->dma_slave);
2967
2968 if (err) {
6db5a8ba 2969 d40_err(base->dev, "Failed to register slave channels\n");
8d318a50
LW		 2970 goto failure1;
2971 }
2972
2973 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
664a57ec 2974 base->num_log_chans, ARRAY_SIZE(dma40_memcpy_channels));
8d318a50
LW		 2975
2976 dma_cap_zero(base->dma_memcpy.cap_mask);
2977 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
7ad74a7c
RV		 2978 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2979
2980 d40_ops_init(base, &base->dma_memcpy);
8d318a50
LW		 2981
2982 err = dma_async_device_register(&base->dma_memcpy);
2983
2984 if (err) {
6db5a8ba
RV		 2985 d40_err(base->dev,
2986 "Failed to regsiter memcpy only channels\n");
8d318a50
LW		 2987 goto failure2;
2988 }
2989
2990 d40_chan_init(base, &base->dma_both, base->phy_chans,
2991 0, num_reserved_chans);
2992
2993 dma_cap_zero(base->dma_both.cap_mask);
2994 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2995 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
7ad74a7c 2996 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
0c842b55 2997 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
7ad74a7c
RV		 2998
2999 d40_ops_init(base, &base->dma_both);
8d318a50
LW		 3000 err = dma_async_device_register(&base->dma_both);
3001
3002 if (err) {
6db5a8ba
RV		 3003 d40_err(base->dev,
3004 "Failed to register logical and physical capable channels\n");
8d318a50
LW		 3005 goto failure3;
3006 }
3007 return 0;
3008failure3:
3009 dma_async_device_unregister(&base->dma_memcpy);
3010failure2:
3011 dma_async_device_unregister(&base->dma_slave);
3012failure1:
3013 return err;
3014}
3015
7fb3e75e
N		 3016/* Suspend resume functionality */
3017#ifdef CONFIG_PM
3018static int dma40_pm_suspend(struct device *dev)
3019{
28c7a19d
N		 3020 struct platform_device *pdev = to_platform_device(dev);
3021 struct d40_base *base = platform_get_drvdata(pdev);
3022 int ret = 0;
7fb3e75e 3023
28c7a19d
N		 3024 if (base->lcpa_regulator)
3025 ret = regulator_disable(base->lcpa_regulator);
3026 return ret;
7fb3e75e
N		 3027}
3028
3029static int dma40_runtime_suspend(struct device *dev)
3030{
3031 struct platform_device *pdev = to_platform_device(dev);
3032 struct d40_base *base = platform_get_drvdata(pdev);
3033
3034 d40_save_restore_registers(base, true);
3035
3036 /* Don't disable/enable clocks for v1 due to HW bugs */
3037 if (base->rev != 1)
3038 writel_relaxed(base->gcc_pwr_off_mask,
3039 base->virtbase + D40_DREG_GCC);
3040
3041 return 0;
3042}
3043
3044static int dma40_runtime_resume(struct device *dev)
3045{
3046 struct platform_device *pdev = to_platform_device(dev);
3047 struct d40_base *base = platform_get_drvdata(pdev);
3048
3049 if (base->initialized)
3050 d40_save_restore_registers(base, false);
3051
3052 writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
3053 base->virtbase + D40_DREG_GCC);
3054 return 0;
3055}
3056
28c7a19d
N		 3057static int dma40_resume(struct device *dev)
3058{
3059 struct platform_device *pdev = to_platform_device(dev);
3060 struct d40_base *base = platform_get_drvdata(pdev);
3061 int ret = 0;
3062
3063 if (base->lcpa_regulator)
3064 ret = regulator_enable(base->lcpa_regulator);
3065
3066 return ret;
3067}
7fb3e75e
N		 3068
3069static const struct dev_pm_ops dma40_pm_ops = {
3070 .suspend = dma40_pm_suspend,
3071 .runtime_suspend = dma40_runtime_suspend,
3072 .runtime_resume = dma40_runtime_resume,
28c7a19d 3073 .resume = dma40_resume,
7fb3e75e
N		 3074};
3075#define DMA40_PM_OPS (&dma40_pm_ops)
3076#else
3077#define DMA40_PM_OPS NULL
3078#endif
3079
8d318a50
LW		 3080/* Initialization functions. */
3081
3082static int __init d40_phy_res_init(struct d40_base *base)
3083{
3084 int i;
3085 int num_phy_chans_avail = 0;
3086 u32 val[2];
3087 int odd_even_bit = -2;
7fb3e75e 3088 int gcc = D40_DREG_GCC_ENA;
8d318a50
LW		 3089
3090 val[0] = readl(base->virtbase + D40_DREG_PRSME);
3091 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
3092
3093 for (i = 0; i < base->num_phy_chans; i++) {
3094 base->phy_res[i].num = i;
3095 odd_even_bit += 2 * ((i % 2) == 0);
3096 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
3097 /* Mark security only channels as occupied */
3098 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
3099 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
7fb3e75e
N		 3100 base->phy_res[i].reserved = true;
3101 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3102 D40_DREG_GCC_SRC);
3103 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3104 D40_DREG_GCC_DST);
3105
3106
8d318a50
LW		 3107 } else {
3108 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
3109 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
7fb3e75e 3110 base->phy_res[i].reserved = false;
8d318a50
LW		 3111 num_phy_chans_avail++;
3112 }
3113 spin_lock_init(&base->phy_res[i].lock);
3114 }
6b7acd84
JA		 3115
3116 /* Mark disabled channels as occupied */
3117 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
f57b407c
RV		 3118 int chan = base->plat_data->disabled_channels[i];
3119
3120 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
3121 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
7fb3e75e
N		 3122 base->phy_res[chan].reserved = true;
3123 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3124 D40_DREG_GCC_SRC);
3125 gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3126 D40_DREG_GCC_DST);
f57b407c 3127 num_phy_chans_avail--;
6b7acd84
JA		 3128 }
3129
7407048b
FB		 3130 /* Mark soft_lli channels */
3131 for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {
3132 int chan = base->plat_data->soft_lli_chans[i];
3133
3134 base->phy_res[chan].use_soft_lli = true;
3135 }
3136
8d318a50
LW		 3137 dev_info(base->dev, "%d of %d physical DMA channels available\n",
3138 num_phy_chans_avail, base->num_phy_chans);
3139
3140 /* Verify settings extended vs standard */
3141 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
3142
3143 for (i = 0; i < base->num_phy_chans; i++) {
3144
3145 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
3146 (val[0] & 0x3) != 1)
3147 dev_info(base->dev,
3148 "[%s] INFO: channel %d is misconfigured (%d)\n",
3149 __func__, i, val[0] & 0x3);
3150
3151 val[0] = val[0] >> 2;
3152 }
3153
7fb3e75e
N		 3154 /*
3155 * To keep things simple, Enable all clocks initially.
3156 * The clocks will get managed later post channel allocation.
3157 * The clocks for the event lines on which reserved channels exists
3158 * are not managed here.
3159 */
3160 writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
3161 base->gcc_pwr_off_mask = gcc;
3162
8d318a50
LW		 3163 return num_phy_chans_avail;
3164}
3165
3166static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
3167{
8d318a50
LW		 3168 struct stedma40_platform_data *plat_data;
3169 struct clk *clk = NULL;
3170 void __iomem *virtbase = NULL;
3171 struct resource *res = NULL;
3172 struct d40_base *base = NULL;
3173 int num_log_chans = 0;
3174 int num_phy_chans;
b707c658 3175 int clk_ret = -EINVAL;
8d318a50 3176 int i;
f4b89764
LW		 3177 u32 pid;
3178 u32 cid;
3179 u8 rev;
8d318a50
LW		 3180
3181 clk = clk_get(&pdev->dev, NULL);
8d318a50 3182 if (IS_ERR(clk)) {
6db5a8ba 3183 d40_err(&pdev->dev, "No matching clock found\n");
8d318a50
LW		 3184 goto failure;
3185 }
3186
b707c658
UH		 3187 clk_ret = clk_prepare_enable(clk);
3188 if (clk_ret) {
3189 d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
3190 goto failure;
3191 }
8d318a50
LW		 3192
3193 /* Get IO for DMAC base address */
3194 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3195 if (!res)
3196 goto failure;
3197
3198 if (request_mem_region(res->start, resource_size(res),
3199 D40_NAME " I/O base") == NULL)
3200 goto failure;
3201
3202 virtbase = ioremap(res->start, resource_size(res));
3203 if (!virtbase)
3204 goto failure;
3205
f4b89764
LW		 3206 /* This is just a regular AMBA PrimeCell ID actually */
3207 for (pid = 0, i = 0; i < 4; i++)
3208 pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i)
3209 & 255) << (i * 8);
3210 for (cid = 0, i = 0; i < 4; i++)
3211 cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i)
3212 & 255) << (i * 8);
8d318a50 3213
f4b89764
LW		 3214 if (cid != AMBA_CID) {
3215 d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
3216 goto failure;
3217 }
3218 if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
6db5a8ba 3219 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
f4b89764
LW		 3220 AMBA_MANF_BITS(pid),
3221 AMBA_VENDOR_ST);
8d318a50
LW		 3222 goto failure;
3223 }
f4b89764
LW		 3224 /*
3225 * HW revision:
3226 * DB8500ed has revision 0
3227 * ? has revision 1
3228 * DB8500v1 has revision 2
3229 * DB8500v2 has revision 3
47db92f4
GB		 3230 * AP9540v1 has revision 4
3231 * DB8540v1 has revision 4
f4b89764
LW		 3232 */
3233 rev = AMBA_REV_BITS(pid);
3ae0267f 3234
47db92f4
GB		 3235 plat_data = pdev->dev.platform_data;
3236
8d318a50 3237 /* The number of physical channels on this HW */
47db92f4
GB		 3238 if (plat_data->num_of_phy_chans)
3239 num_phy_chans = plat_data->num_of_phy_chans;
3240 else
3241 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
8d318a50 3242
47db92f4
GB		 3243 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x with %d physical channels\n",
3244 rev, res->start, num_phy_chans);
8d318a50 3245
1bdae6f4
N		 3246 if (rev < 2) {
3247 d40_err(&pdev->dev, "hardware revision: %d is not supported",
3248 rev);
3249 goto failure;
3250 }
3251
8d318a50
LW		 3252 /* Count the number of logical channels in use */
3253 for (i = 0; i < plat_data->dev_len; i++)
3254 if (plat_data->dev_rx[i] != 0)
3255 num_log_chans++;
3256
3257 for (i = 0; i < plat_data->dev_len; i++)
3258 if (plat_data->dev_tx[i] != 0)
3259 num_log_chans++;
3260
3261 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
664a57ec 3262 (num_phy_chans + num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) *
8d318a50
LW		 3263 sizeof(struct d40_chan), GFP_KERNEL);
3264
3265 if (base == NULL) {
6db5a8ba 3266 d40_err(&pdev->dev, "Out of memory\n");
8d318a50
LW		 3267 goto failure;
3268 }
3269
3ae0267f 3270 base->rev = rev;
8d318a50
LW		 3271 base->clk = clk;
3272 base->num_phy_chans = num_phy_chans;
3273 base->num_log_chans = num_log_chans;
3274 base->phy_start = res->start;
3275 base->phy_size = resource_size(res);
3276 base->virtbase = virtbase;
3277 base->plat_data = plat_data;
3278 base->dev = &pdev->dev;
3279 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
3280 base->log_chans = &base->phy_chans[num_phy_chans];
3281
3cb645dc
TL		 3282 if (base->plat_data->num_of_phy_chans == 14) {
3283 base->gen_dmac.backup = d40_backup_regs_v4b;
3284 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;
3285 base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;
3286 base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;
3287 base->gen_dmac.realtime_en = D40_DREG_CRSEG1;
3288 base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;
3289 base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;
3290 base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;
3291 base->gen_dmac.il = il_v4b;
3292 base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);
3293 base->gen_dmac.init_reg = dma_init_reg_v4b;
3294 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);
3295 } else {
3296 if (base->rev >= 3) {
3297 base->gen_dmac.backup = d40_backup_regs_v4a;
3298 base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;
3299 }
3300 base->gen_dmac.interrupt_en = D40_DREG_PCMIS;
3301 base->gen_dmac.interrupt_clear = D40_DREG_PCICR;
3302 base->gen_dmac.realtime_en = D40_DREG_RSEG1;
3303 base->gen_dmac.realtime_clear = D40_DREG_RCEG1;
3304 base->gen_dmac.high_prio_en = D40_DREG_PSEG1;
3305 base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;
3306 base->gen_dmac.il = il_v4a;
3307 base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);
3308 base->gen_dmac.init_reg = dma_init_reg_v4a;
3309 base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
3310 }
3311
8d318a50
LW		 3312 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
3313 GFP_KERNEL);
3314 if (!base->phy_res)
3315 goto failure;
3316
3317 base->lookup_phy_chans = kzalloc(num_phy_chans *
3318 sizeof(struct d40_chan *),
3319 GFP_KERNEL);
3320 if (!base->lookup_phy_chans)
3321 goto failure;
3322
664a57ec 3323 if (num_log_chans + ARRAY_SIZE(dma40_memcpy_channels)) {
8d318a50
LW		 3324 /*
3325 * The max number of logical channels are event lines for all
3326 * src devices and dst devices
3327 */
3328 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
3329 sizeof(struct d40_chan *),
3330 GFP_KERNEL);
3331 if (!base->lookup_log_chans)
3332 goto failure;
3333 }
698e4732 3334
7fb3e75e
N		 3335 base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
3336 sizeof(d40_backup_regs_chan),
8d318a50 3337 GFP_KERNEL);
7fb3e75e
N		 3338 if (!base->reg_val_backup_chan)
3339 goto failure;
3340
3341 base->lcla_pool.alloc_map =
3342 kzalloc(num_phy_chans * sizeof(struct d40_desc *)
3343 * D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
8d318a50
LW		 3344 if (!base->lcla_pool.alloc_map)
3345 goto failure;
3346
c675b1b4
JA		 3347 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
3348 0, SLAB_HWCACHE_ALIGN,
3349 NULL);
3350 if (base->desc_slab == NULL)
3351 goto failure;
3352
8d318a50
LW		 3353 return base;
3354
3355failure:
b707c658
UH		 3356 if (!clk_ret)
3357 clk_disable_unprepare(clk);
3358 if (!IS_ERR(clk))
8d318a50 3359 clk_put(clk);
8d318a50
LW		 3360 if (virtbase)
3361 iounmap(virtbase);
3362 if (res)
3363 release_mem_region(res->start,
3364 resource_size(res));
3365 if (virtbase)
3366 iounmap(virtbase);
3367
3368 if (base) {
3369 kfree(base->lcla_pool.alloc_map);
1bdae6f4 3370 kfree(base->reg_val_backup_chan);
8d318a50
LW		 3371 kfree(base->lookup_log_chans);
3372 kfree(base->lookup_phy_chans);
3373 kfree(base->phy_res);
3374 kfree(base);
3375 }
3376
3377 return NULL;
3378}
3379
3380static void __init d40_hw_init(struct d40_base *base)
3381{
3382
8d318a50
LW		 3383 int i;
3384 u32 prmseo[2] = {0, 0};
3385 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3386 u32 pcmis = 0;
3387 u32 pcicr = 0;
3cb645dc
TL		 3388 struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;
3389 u32 reg_size = base->gen_dmac.init_reg_size;
8d318a50 3390
3cb645dc 3391 for (i = 0; i < reg_size; i++)
8d318a50
LW		 3392 writel(dma_init_reg[i].val,
3393 base->virtbase + dma_init_reg[i].reg);
3394
3395 /* Configure all our dma channels to default settings */
3396 for (i = 0; i < base->num_phy_chans; i++) {
3397
3398 activeo[i % 2] = activeo[i % 2] << 2;
3399
3400 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
3401 == D40_ALLOC_PHY) {
3402 activeo[i % 2] |= 3;
3403 continue;
3404 }
3405
3406 /* Enable interrupt # */
3407 pcmis = (pcmis << 1) | 1;
3408
3409 /* Clear interrupt # */
3410 pcicr = (pcicr << 1) | 1;
3411
3412 /* Set channel to physical mode */
3413 prmseo[i % 2] = prmseo[i % 2] << 2;
3414 prmseo[i % 2] |= 1;
3415
3416 }
3417
3418 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
3419 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
3420 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
3421 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
3422
3423 /* Write which interrupt to enable */
3cb645dc 3424 writel(pcmis, base->virtbase + base->gen_dmac.interrupt_en);
8d318a50
LW		 3425
3426 /* Write which interrupt to clear */
3cb645dc 3427 writel(pcicr, base->virtbase + base->gen_dmac.interrupt_clear);
8d318a50 3428
3cb645dc
TL		 3429 /* These are __initdata and cannot be accessed after init */
3430 base->gen_dmac.init_reg = NULL;
3431 base->gen_dmac.init_reg_size = 0;
8d318a50
LW		 3432}
3433
508849ad
LW		 3434static int __init d40_lcla_allocate(struct d40_base *base)
3435{
026cbc42 3436 struct d40_lcla_pool *pool = &base->lcla_pool;
508849ad
LW		 3437 unsigned long *page_list;
3438 int i, j;
3439 int ret = 0;
3440
3441 /*
3442 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3443 * To full fill this hardware requirement without wasting 256 kb
3444 * we allocate pages until we get an aligned one.
3445 */
3446 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
3447 GFP_KERNEL);
3448
3449 if (!page_list) {
3450 ret = -ENOMEM;
3451 goto failure;
3452 }
3453
3454 /* Calculating how many pages that are required */
3455 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
3456
3457 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
3458 page_list[i] = __get_free_pages(GFP_KERNEL,
3459 base->lcla_pool.pages);
3460 if (!page_list[i]) {
3461
6db5a8ba
RV		 3462 d40_err(base->dev, "Failed to allocate %d pages.\n",
3463 base->lcla_pool.pages);
508849ad
LW		 3464
3465 for (j = 0; j < i; j++)
3466 free_pages(page_list[j], base->lcla_pool.pages);
3467 goto failure;
3468 }
3469
3470 if ((virt_to_phys((void *)page_list[i]) &
3471 (LCLA_ALIGNMENT - 1)) == 0)
3472 break;
3473 }
3474
3475 for (j = 0; j < i; j++)
3476 free_pages(page_list[j], base->lcla_pool.pages);
3477
3478 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
3479 base->lcla_pool.base = (void *)page_list[i];
3480 } else {
767a9675
JA		 3481 /*
3482 * After many attempts and no succees with finding the correct
3483 * alignment, try with allocating a big buffer.
3484 */
508849ad
LW		 3485 dev_warn(base->dev,
3486 "[%s] Failed to get %d pages @ 18 bit align.\n",
3487 __func__, base->lcla_pool.pages);
3488 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
3489 base->num_phy_chans +
3490 LCLA_ALIGNMENT,
3491 GFP_KERNEL);
3492 if (!base->lcla_pool.base_unaligned) {
3493 ret = -ENOMEM;
3494 goto failure;
3495 }
3496
3497 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
3498 LCLA_ALIGNMENT);
3499 }
3500
026cbc42
RV		 3501 pool->dma_addr = dma_map_single(base->dev, pool->base,
3502 SZ_1K * base->num_phy_chans,
3503 DMA_TO_DEVICE);
3504 if (dma_mapping_error(base->dev, pool->dma_addr)) {
3505 pool->dma_addr = 0;
3506 ret = -ENOMEM;
3507 goto failure;
3508 }
3509
508849ad
LW		 3510 writel(virt_to_phys(base->lcla_pool.base),
3511 base->virtbase + D40_DREG_LCLA);
3512failure:
3513 kfree(page_list);
3514 return ret;
3515}
3516
8d318a50
LW		 3517static int __init d40_probe(struct platform_device *pdev)
3518{
3519 int err;
3520 int ret = -ENOENT;
3521 struct d40_base *base;
3522 struct resource *res = NULL;
3523 int num_reserved_chans;
3524 u32 val;
3525
3526 base = d40_hw_detect_init(pdev);
3527
3528 if (!base)
3529 goto failure;
3530
3531 num_reserved_chans = d40_phy_res_init(base);
3532
3533 platform_set_drvdata(pdev, base);
3534
3535 spin_lock_init(&base->interrupt_lock);
3536 spin_lock_init(&base->execmd_lock);
3537
3538 /* Get IO for logical channel parameter address */
3539 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
3540 if (!res) {
3541 ret = -ENOENT;
6db5a8ba 3542 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
8d318a50
LW		 3543 goto failure;
3544 }
3545 base->lcpa_size = resource_size(res);
3546 base->phy_lcpa = res->start;
3547
3548 if (request_mem_region(res->start, resource_size(res),
3549 D40_NAME " I/O lcpa") == NULL) {
3550 ret = -EBUSY;
6db5a8ba
RV		 3551 d40_err(&pdev->dev,
3552 "Failed to request LCPA region 0x%x-0x%x\n",
3553 res->start, res->end);
8d318a50
LW		 3554 goto failure;
3555 }
3556
3557 /* We make use of ESRAM memory for this. */
3558 val = readl(base->virtbase + D40_DREG_LCPA);
3559 if (res->start != val && val != 0) {
3560 dev_warn(&pdev->dev,
3561 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
3562 __func__, val, res->start);
3563 } else
3564 writel(res->start, base->virtbase + D40_DREG_LCPA);
3565
3566 base->lcpa_base = ioremap(res->start, resource_size(res));
3567 if (!base->lcpa_base) {
3568 ret = -ENOMEM;
6db5a8ba 3569 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
8d318a50
LW		 3570 goto failure;
3571 }
28c7a19d
N		 3572 /* If lcla has to be located in ESRAM we don't need to allocate */
3573 if (base->plat_data->use_esram_lcla) {
3574 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
3575 "lcla_esram");
3576 if (!res) {
3577 ret = -ENOENT;
3578 d40_err(&pdev->dev,
3579 "No \"lcla_esram\" memory resource\n");
3580 goto failure;
3581 }
3582 base->lcla_pool.base = ioremap(res->start,
3583 resource_size(res));
3584 if (!base->lcla_pool.base) {
3585 ret = -ENOMEM;
3586 d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
3587 goto failure;
3588 }
3589 writel(res->start, base->virtbase + D40_DREG_LCLA);
8d318a50 3590
28c7a19d
N		 3591 } else {
3592 ret = d40_lcla_allocate(base);
3593 if (ret) {
3594 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
3595 goto failure;
3596 }
8d318a50
LW		 3597 }
3598
3599 spin_lock_init(&base->lcla_pool.lock);
3600
8d318a50
LW		 3601 base->irq = platform_get_irq(pdev, 0);
3602
3603 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
8d318a50 3604 if (ret) {
6db5a8ba 3605 d40_err(&pdev->dev, "No IRQ defined\n");
8d318a50
LW		 3606 goto failure;
3607 }
3608
7fb3e75e
N		 3609 pm_runtime_irq_safe(base->dev);
3610 pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);
3611 pm_runtime_use_autosuspend(base->dev);
3612 pm_runtime_enable(base->dev);
3613 pm_runtime_resume(base->dev);
28c7a19d
N		 3614
3615 if (base->plat_data->use_esram_lcla) {
3616
3617 base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");
3618 if (IS_ERR(base->lcpa_regulator)) {
3619 d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
3620 base->lcpa_regulator = NULL;
3621 goto failure;
3622 }
3623
3624 ret = regulator_enable(base->lcpa_regulator);
3625 if (ret) {
3626 d40_err(&pdev->dev,
3627 "Failed to enable lcpa_regulator\n");
3628 regulator_put(base->lcpa_regulator);
3629 base->lcpa_regulator = NULL;
3630 goto failure;
3631 }
3632 }
3633
7fb3e75e 3634 base->initialized = true;
8d318a50
LW		 3635 err = d40_dmaengine_init(base, num_reserved_chans);
3636 if (err)
3637 goto failure;
3638
b96710e5
PF		 3639 base->dev->dma_parms = &base->dma_parms;
3640 err = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);
3641 if (err) {
3642 d40_err(&pdev->dev, "Failed to set dma max seg size\n");
3643 goto failure;
3644 }
3645
8d318a50
LW		 3646 d40_hw_init(base);
3647
3648 dev_info(base->dev, "initialized\n");
3649 return 0;
3650
3651failure:
3652 if (base) {
c675b1b4
JA		 3653 if (base->desc_slab)
3654 kmem_cache_destroy(base->desc_slab);
8d318a50
LW		 3655 if (base->virtbase)
3656 iounmap(base->virtbase);
026cbc42 3657
28c7a19d
N		 3658 if (base->lcla_pool.base && base->plat_data->use_esram_lcla) {
3659 iounmap(base->lcla_pool.base);
3660 base->lcla_pool.base = NULL;
3661 }
3662
026cbc42
RV		 3663 if (base->lcla_pool.dma_addr)
3664 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
3665 SZ_1K * base->num_phy_chans,
3666 DMA_TO_DEVICE);
3667
508849ad
LW		 3668 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3669 free_pages((unsigned long)base->lcla_pool.base,
3670 base->lcla_pool.pages);
767a9675
JA		 3671
3672 kfree(base->lcla_pool.base_unaligned);
3673
8d318a50
LW		 3674 if (base->phy_lcpa)
3675 release_mem_region(base->phy_lcpa,
3676 base->lcpa_size);
3677 if (base->phy_start)
3678 release_mem_region(base->phy_start,
3679 base->phy_size);
3680 if (base->clk) {
da2ac56a 3681 clk_disable_unprepare(base->clk);
8d318a50
LW		 3682 clk_put(base->clk);
3683 }
3684
28c7a19d
N		 3685 if (base->lcpa_regulator) {
3686 regulator_disable(base->lcpa_regulator);
3687 regulator_put(base->lcpa_regulator);
3688 }
3689
8d318a50
LW		 3690 kfree(base->lcla_pool.alloc_map);
3691 kfree(base->lookup_log_chans);
3692 kfree(base->lookup_phy_chans);
3693 kfree(base->phy_res);
3694 kfree(base);
3695 }
3696
6db5a8ba 3697 d40_err(&pdev->dev, "probe failed\n");
8d318a50
LW		 3698 return ret;
3699}
3700
3701static struct platform_driver d40_driver = {
3702 .driver = {
3703 .owner = THIS_MODULE,
3704 .name = D40_NAME,
7fb3e75e 3705 .pm = DMA40_PM_OPS,
8d318a50
LW		 3706 },
3707};
3708
cb9ab2d8 3709static int __init stedma40_init(void)
8d318a50
LW		 3710{
3711 return platform_driver_probe(&d40_driver, d40_probe);
3712}
a0eb221a 3713subsys_initcall(stedma40_init);
ubuntu-bionic-kernel mirror