2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
6 * License terms: GNU General Public License (GPL) version 2
9 #include <linux/dma-mapping.h>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/dmaengine.h>
14 #include <linux/platform_device.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/log2.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/err.h>
22 #include <linux/of_dma.h>
23 #include <linux/amba/bus.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/platform_data/dma-ste-dma40.h>
27 #include "dmaengine.h"
28 #include "ste_dma40_ll.h"
30 #define D40_NAME "dma40"
32 #define D40_PHY_CHAN -1
34 /* For masking out/in 2 bit channel positions */
35 #define D40_CHAN_POS(chan) (2 * (chan / 2))
36 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
38 /* Maximum iterations taken before giving up suspending a channel */
39 #define D40_SUSPEND_MAX_IT 500
42 #define DMA40_AUTOSUSPEND_DELAY 100
44 /* Hardware requirement on LCLA alignment */
45 #define LCLA_ALIGNMENT 0x40000
47 /* Max number of links per event group */
48 #define D40_LCLA_LINK_PER_EVENT_GRP 128
49 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
51 /* Max number of logical channels per physical channel */
52 #define D40_MAX_LOG_CHAN_PER_PHY 32
54 /* Attempts before giving up to trying to get pages that are aligned */
55 #define MAX_LCLA_ALLOC_ATTEMPTS 256
57 /* Bit markings for allocation map */
58 #define D40_ALLOC_FREE BIT(31)
59 #define D40_ALLOC_PHY BIT(30)
60 #define D40_ALLOC_LOG_FREE 0
62 #define D40_MEMCPY_MAX_CHANS 8
64 /* Reserved event lines for memcpy only. */
65 #define DB8500_DMA_MEMCPY_EV_0 51
66 #define DB8500_DMA_MEMCPY_EV_1 56
67 #define DB8500_DMA_MEMCPY_EV_2 57
68 #define DB8500_DMA_MEMCPY_EV_3 58
69 #define DB8500_DMA_MEMCPY_EV_4 59
70 #define DB8500_DMA_MEMCPY_EV_5 60
72 static int dma40_memcpy_channels
[] = {
73 DB8500_DMA_MEMCPY_EV_0
,
74 DB8500_DMA_MEMCPY_EV_1
,
75 DB8500_DMA_MEMCPY_EV_2
,
76 DB8500_DMA_MEMCPY_EV_3
,
77 DB8500_DMA_MEMCPY_EV_4
,
78 DB8500_DMA_MEMCPY_EV_5
,
81 /* Default configuration for physcial memcpy */
82 static struct stedma40_chan_cfg dma40_memcpy_conf_phy
= {
83 .mode
= STEDMA40_MODE_PHYSICAL
,
84 .dir
= DMA_MEM_TO_MEM
,
86 .src_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
87 .src_info
.psize
= STEDMA40_PSIZE_PHY_1
,
88 .src_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
90 .dst_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
91 .dst_info
.psize
= STEDMA40_PSIZE_PHY_1
,
92 .dst_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
95 /* Default configuration for logical memcpy */
96 static struct stedma40_chan_cfg dma40_memcpy_conf_log
= {
97 .mode
= STEDMA40_MODE_LOGICAL
,
98 .dir
= DMA_MEM_TO_MEM
,
100 .src_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
101 .src_info
.psize
= STEDMA40_PSIZE_LOG_1
,
102 .src_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
104 .dst_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
105 .dst_info
.psize
= STEDMA40_PSIZE_LOG_1
,
106 .dst_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
110 * enum 40_command - The different commands and/or statuses.
112 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
113 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
114 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
115 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
120 D40_DMA_SUSPEND_REQ
= 2,
121 D40_DMA_SUSPENDED
= 3
125 * enum d40_events - The different Event Enables for the event lines.
127 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
128 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
129 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
130 * @D40_ROUND_EVENTLINE: Status check for event line.
134 D40_DEACTIVATE_EVENTLINE
= 0,
135 D40_ACTIVATE_EVENTLINE
= 1,
136 D40_SUSPEND_REQ_EVENTLINE
= 2,
137 D40_ROUND_EVENTLINE
= 3
141 * These are the registers that has to be saved and later restored
142 * when the DMA hw is powered off.
143 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
145 static u32 d40_backup_regs
[] = {
154 #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
157 * since 9540 and 8540 has the same HW revision
158 * use v4a for 9540 or ealier
159 * use v4b for 8540 or later
161 * DB8500ed has revision 0
162 * DB8500v1 has revision 2
163 * DB8500v2 has revision 3
164 * AP9540v1 has revision 4
165 * DB8540v1 has revision 4
166 * TODO: Check if all these registers have to be saved/restored on dma40 v4a
168 static u32 d40_backup_regs_v4a
[] = {
187 #define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
189 static u32 d40_backup_regs_v4b
[] = {
212 #define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
214 static u32 d40_backup_regs_chan
[] = {
225 #define BACKUP_REGS_SZ_MAX ((BACKUP_REGS_SZ_V4A > BACKUP_REGS_SZ_V4B) ? \
226 BACKUP_REGS_SZ_V4A : BACKUP_REGS_SZ_V4B)
229 * struct d40_interrupt_lookup - lookup table for interrupt handler
231 * @src: Interrupt mask register.
232 * @clr: Interrupt clear register.
233 * @is_error: true if this is an error interrupt.
234 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
235 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
237 struct d40_interrupt_lookup
{
245 static struct d40_interrupt_lookup il_v4a
[] = {
246 {D40_DREG_LCTIS0
, D40_DREG_LCICR0
, false, 0},
247 {D40_DREG_LCTIS1
, D40_DREG_LCICR1
, false, 32},
248 {D40_DREG_LCTIS2
, D40_DREG_LCICR2
, false, 64},
249 {D40_DREG_LCTIS3
, D40_DREG_LCICR3
, false, 96},
250 {D40_DREG_LCEIS0
, D40_DREG_LCICR0
, true, 0},
251 {D40_DREG_LCEIS1
, D40_DREG_LCICR1
, true, 32},
252 {D40_DREG_LCEIS2
, D40_DREG_LCICR2
, true, 64},
253 {D40_DREG_LCEIS3
, D40_DREG_LCICR3
, true, 96},
254 {D40_DREG_PCTIS
, D40_DREG_PCICR
, false, D40_PHY_CHAN
},
255 {D40_DREG_PCEIS
, D40_DREG_PCICR
, true, D40_PHY_CHAN
},
258 static struct d40_interrupt_lookup il_v4b
[] = {
259 {D40_DREG_CLCTIS1
, D40_DREG_CLCICR1
, false, 0},
260 {D40_DREG_CLCTIS2
, D40_DREG_CLCICR2
, false, 32},
261 {D40_DREG_CLCTIS3
, D40_DREG_CLCICR3
, false, 64},
262 {D40_DREG_CLCTIS4
, D40_DREG_CLCICR4
, false, 96},
263 {D40_DREG_CLCTIS5
, D40_DREG_CLCICR5
, false, 128},
264 {D40_DREG_CLCEIS1
, D40_DREG_CLCICR1
, true, 0},
265 {D40_DREG_CLCEIS2
, D40_DREG_CLCICR2
, true, 32},
266 {D40_DREG_CLCEIS3
, D40_DREG_CLCICR3
, true, 64},
267 {D40_DREG_CLCEIS4
, D40_DREG_CLCICR4
, true, 96},
268 {D40_DREG_CLCEIS5
, D40_DREG_CLCICR5
, true, 128},
269 {D40_DREG_CPCTIS
, D40_DREG_CPCICR
, false, D40_PHY_CHAN
},
270 {D40_DREG_CPCEIS
, D40_DREG_CPCICR
, true, D40_PHY_CHAN
},
274 * struct d40_reg_val - simple lookup struct
276 * @reg: The register.
277 * @val: The value that belongs to the register in reg.
284 static __initdata
struct d40_reg_val dma_init_reg_v4a
[] = {
285 /* Clock every part of the DMA block from start */
286 { .reg
= D40_DREG_GCC
, .val
= D40_DREG_GCC_ENABLE_ALL
},
288 /* Interrupts on all logical channels */
289 { .reg
= D40_DREG_LCMIS0
, .val
= 0xFFFFFFFF},
290 { .reg
= D40_DREG_LCMIS1
, .val
= 0xFFFFFFFF},
291 { .reg
= D40_DREG_LCMIS2
, .val
= 0xFFFFFFFF},
292 { .reg
= D40_DREG_LCMIS3
, .val
= 0xFFFFFFFF},
293 { .reg
= D40_DREG_LCICR0
, .val
= 0xFFFFFFFF},
294 { .reg
= D40_DREG_LCICR1
, .val
= 0xFFFFFFFF},
295 { .reg
= D40_DREG_LCICR2
, .val
= 0xFFFFFFFF},
296 { .reg
= D40_DREG_LCICR3
, .val
= 0xFFFFFFFF},
297 { .reg
= D40_DREG_LCTIS0
, .val
= 0xFFFFFFFF},
298 { .reg
= D40_DREG_LCTIS1
, .val
= 0xFFFFFFFF},
299 { .reg
= D40_DREG_LCTIS2
, .val
= 0xFFFFFFFF},
300 { .reg
= D40_DREG_LCTIS3
, .val
= 0xFFFFFFFF}
302 static __initdata
struct d40_reg_val dma_init_reg_v4b
[] = {
303 /* Clock every part of the DMA block from start */
304 { .reg
= D40_DREG_GCC
, .val
= D40_DREG_GCC_ENABLE_ALL
},
306 /* Interrupts on all logical channels */
307 { .reg
= D40_DREG_CLCMIS1
, .val
= 0xFFFFFFFF},
308 { .reg
= D40_DREG_CLCMIS2
, .val
= 0xFFFFFFFF},
309 { .reg
= D40_DREG_CLCMIS3
, .val
= 0xFFFFFFFF},
310 { .reg
= D40_DREG_CLCMIS4
, .val
= 0xFFFFFFFF},
311 { .reg
= D40_DREG_CLCMIS5
, .val
= 0xFFFFFFFF},
312 { .reg
= D40_DREG_CLCICR1
, .val
= 0xFFFFFFFF},
313 { .reg
= D40_DREG_CLCICR2
, .val
= 0xFFFFFFFF},
314 { .reg
= D40_DREG_CLCICR3
, .val
= 0xFFFFFFFF},
315 { .reg
= D40_DREG_CLCICR4
, .val
= 0xFFFFFFFF},
316 { .reg
= D40_DREG_CLCICR5
, .val
= 0xFFFFFFFF},
317 { .reg
= D40_DREG_CLCTIS1
, .val
= 0xFFFFFFFF},
318 { .reg
= D40_DREG_CLCTIS2
, .val
= 0xFFFFFFFF},
319 { .reg
= D40_DREG_CLCTIS3
, .val
= 0xFFFFFFFF},
320 { .reg
= D40_DREG_CLCTIS4
, .val
= 0xFFFFFFFF},
321 { .reg
= D40_DREG_CLCTIS5
, .val
= 0xFFFFFFFF}
325 * struct d40_lli_pool - Structure for keeping LLIs in memory
327 * @base: Pointer to memory area when the pre_alloc_lli's are not large
328 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
329 * pre_alloc_lli is used.
330 * @dma_addr: DMA address, if mapped
331 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
332 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
333 * one buffer to one buffer.
335 struct d40_lli_pool
{
339 /* Space for dst and src, plus an extra for padding */
340 u8 pre_alloc_lli
[3 * sizeof(struct d40_phy_lli
)];
344 * struct d40_desc - A descriptor is one DMA job.
346 * @lli_phy: LLI settings for physical channel. Both src and dst=
347 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
348 * lli_len equals one.
349 * @lli_log: Same as above but for logical channels.
350 * @lli_pool: The pool with two entries pre-allocated.
351 * @lli_len: Number of llis of current descriptor.
352 * @lli_current: Number of transferred llis.
353 * @lcla_alloc: Number of LCLA entries allocated.
354 * @txd: DMA engine struct. Used for among other things for communication
357 * @is_in_client_list: true if the client owns this descriptor.
358 * @cyclic: true if this is a cyclic job
360 * This descriptor is used for both logical and physical transfers.
364 struct d40_phy_lli_bidir lli_phy
;
366 struct d40_log_lli_bidir lli_log
;
368 struct d40_lli_pool lli_pool
;
373 struct dma_async_tx_descriptor txd
;
374 struct list_head node
;
376 bool is_in_client_list
;
381 * struct d40_lcla_pool - LCLA pool settings and data.
383 * @base: The virtual address of LCLA. 18 bit aligned.
384 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
385 * This pointer is only there for clean-up on error.
386 * @pages: The number of pages needed for all physical channels.
387 * Only used later for clean-up on error
388 * @lock: Lock to protect the content in this struct.
389 * @alloc_map: big map over which LCLA entry is own by which job.
391 struct d40_lcla_pool
{
394 void *base_unaligned
;
397 struct d40_desc
**alloc_map
;
401 * struct d40_phy_res - struct for handling eventlines mapped to physical
404 * @lock: A lock protection this entity.
405 * @reserved: True if used by secure world or otherwise.
406 * @num: The physical channel number of this entity.
407 * @allocated_src: Bit mapped to show which src event line's are mapped to
408 * this physical channel. Can also be free or physically allocated.
409 * @allocated_dst: Same as for src but is dst.
410 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
412 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
426 * struct d40_chan - Struct that describes a channel.
428 * @lock: A spinlock to protect this struct.
429 * @log_num: The logical number, if any of this channel.
430 * @pending_tx: The number of pending transfers. Used between interrupt handler
432 * @busy: Set to true when transfer is ongoing on this channel.
433 * @phy_chan: Pointer to physical channel which this instance runs on. If this
434 * point is NULL, then the channel is not allocated.
435 * @chan: DMA engine handle.
436 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
437 * transfer and call client callback.
438 * @client: Cliented owned descriptor list.
439 * @pending_queue: Submitted jobs, to be issued by issue_pending()
440 * @active: Active descriptor.
441 * @done: Completed jobs
442 * @queue: Queued jobs.
443 * @prepare_queue: Prepared jobs.
444 * @dma_cfg: The client configuration of this dma channel.
445 * @configured: whether the dma_cfg configuration is valid
446 * @base: Pointer to the device instance struct.
447 * @src_def_cfg: Default cfg register setting for src.
448 * @dst_def_cfg: Default cfg register setting for dst.
449 * @log_def: Default logical channel settings.
450 * @lcpa: Pointer to dst and src lcpa settings.
451 * @runtime_addr: runtime configured address.
452 * @runtime_direction: runtime configured direction.
454 * This struct can either "be" a logical or a physical channel.
461 struct d40_phy_res
*phy_chan
;
462 struct dma_chan chan
;
463 struct tasklet_struct tasklet
;
464 struct list_head client
;
465 struct list_head pending_queue
;
466 struct list_head active
;
467 struct list_head done
;
468 struct list_head queue
;
469 struct list_head prepare_queue
;
470 struct stedma40_chan_cfg dma_cfg
;
472 struct d40_base
*base
;
473 /* Default register configurations */
476 struct d40_def_lcsp log_def
;
477 struct d40_log_lli_full
*lcpa
;
478 /* Runtime reconfiguration */
479 dma_addr_t runtime_addr
;
480 enum dma_transfer_direction runtime_direction
;
484 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
487 * @backup: the pointer to the registers address array for backup
488 * @backup_size: the size of the registers address array for backup
489 * @realtime_en: the realtime enable register
490 * @realtime_clear: the realtime clear register
491 * @high_prio_en: the high priority enable register
492 * @high_prio_clear: the high priority clear register
493 * @interrupt_en: the interrupt enable register
494 * @interrupt_clear: the interrupt clear register
495 * @il: the pointer to struct d40_interrupt_lookup
496 * @il_size: the size of d40_interrupt_lookup array
497 * @init_reg: the pointer to the struct d40_reg_val
498 * @init_reg_size: the size of d40_reg_val array
500 struct d40_gen_dmac
{
509 struct d40_interrupt_lookup
*il
;
511 struct d40_reg_val
*init_reg
;
516 * struct d40_base - The big global struct, one for each probe'd instance.
518 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
519 * @execmd_lock: Lock for execute command usage since several channels share
520 * the same physical register.
521 * @dev: The device structure.
522 * @virtbase: The virtual base address of the DMA's register.
523 * @rev: silicon revision detected.
524 * @clk: Pointer to the DMA clock structure.
525 * @phy_start: Physical memory start of the DMA registers.
526 * @phy_size: Size of the DMA register map.
527 * @irq: The IRQ number.
528 * @num_memcpy_chans: The number of channels used for memcpy (mem-to-mem
530 * @num_phy_chans: The number of physical channels. Read from HW. This
531 * is the number of available channels for this driver, not counting "Secure
532 * mode" allocated physical channels.
533 * @num_log_chans: The number of logical channels. Calculated from
535 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
536 * @dma_slave: dma_device channels that can do only do slave transfers.
537 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
538 * @phy_chans: Room for all possible physical channels in system.
539 * @log_chans: Room for all possible logical channels in system.
540 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
541 * to log_chans entries.
542 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
543 * to phy_chans entries.
544 * @plat_data: Pointer to provided platform_data which is the driver
546 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
547 * @phy_res: Vector containing all physical channels.
548 * @lcla_pool: lcla pool settings and data.
549 * @lcpa_base: The virtual mapped address of LCPA.
550 * @phy_lcpa: The physical address of the LCPA.
551 * @lcpa_size: The size of the LCPA area.
552 * @desc_slab: cache for descriptors.
553 * @reg_val_backup: Here the values of some hardware registers are stored
554 * before the DMA is powered off. They are restored when the power is back on.
555 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
557 * @reg_val_backup_chan: Backup data for standard channel parameter registers.
558 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
559 * @gen_dmac: the struct for generic registers values to represent u8500/8540
563 spinlock_t interrupt_lock
;
564 spinlock_t execmd_lock
;
566 void __iomem
*virtbase
;
569 phys_addr_t phy_start
;
570 resource_size_t phy_size
;
572 int num_memcpy_chans
;
575 struct device_dma_parameters dma_parms
;
576 struct dma_device dma_both
;
577 struct dma_device dma_slave
;
578 struct dma_device dma_memcpy
;
579 struct d40_chan
*phy_chans
;
580 struct d40_chan
*log_chans
;
581 struct d40_chan
**lookup_log_chans
;
582 struct d40_chan
**lookup_phy_chans
;
583 struct stedma40_platform_data
*plat_data
;
584 struct regulator
*lcpa_regulator
;
585 /* Physical half channels */
586 struct d40_phy_res
*phy_res
;
587 struct d40_lcla_pool lcla_pool
;
590 resource_size_t lcpa_size
;
591 struct kmem_cache
*desc_slab
;
592 u32 reg_val_backup
[BACKUP_REGS_SZ
];
593 u32 reg_val_backup_v4
[BACKUP_REGS_SZ_MAX
];
594 u32
*reg_val_backup_chan
;
595 u16 gcc_pwr_off_mask
;
596 struct d40_gen_dmac gen_dmac
;
599 static struct device
*chan2dev(struct d40_chan
*d40c
)
601 return &d40c
->chan
.dev
->device
;
604 static bool chan_is_physical(struct d40_chan
*chan
)
606 return chan
->log_num
== D40_PHY_CHAN
;
609 static bool chan_is_logical(struct d40_chan
*chan
)
611 return !chan_is_physical(chan
);
614 static void __iomem
*chan_base(struct d40_chan
*chan
)
616 return chan
->base
->virtbase
+ D40_DREG_PCBASE
+
617 chan
->phy_chan
->num
* D40_DREG_PCDELTA
;
620 #define d40_err(dev, format, arg...) \
621 dev_err(dev, "[%s] " format, __func__, ## arg)
623 #define chan_err(d40c, format, arg...) \
624 d40_err(chan2dev(d40c), format, ## arg)
626 static int d40_pool_lli_alloc(struct d40_chan
*d40c
, struct d40_desc
*d40d
,
629 bool is_log
= chan_is_logical(d40c
);
634 align
= sizeof(struct d40_log_lli
);
636 align
= sizeof(struct d40_phy_lli
);
639 base
= d40d
->lli_pool
.pre_alloc_lli
;
640 d40d
->lli_pool
.size
= sizeof(d40d
->lli_pool
.pre_alloc_lli
);
641 d40d
->lli_pool
.base
= NULL
;
643 d40d
->lli_pool
.size
= lli_len
* 2 * align
;
645 base
= kmalloc(d40d
->lli_pool
.size
+ align
, GFP_NOWAIT
);
646 d40d
->lli_pool
.base
= base
;
648 if (d40d
->lli_pool
.base
== NULL
)
653 d40d
->lli_log
.src
= PTR_ALIGN(base
, align
);
654 d40d
->lli_log
.dst
= d40d
->lli_log
.src
+ lli_len
;
656 d40d
->lli_pool
.dma_addr
= 0;
658 d40d
->lli_phy
.src
= PTR_ALIGN(base
, align
);
659 d40d
->lli_phy
.dst
= d40d
->lli_phy
.src
+ lli_len
;
661 d40d
->lli_pool
.dma_addr
= dma_map_single(d40c
->base
->dev
,
666 if (dma_mapping_error(d40c
->base
->dev
,
667 d40d
->lli_pool
.dma_addr
)) {
668 kfree(d40d
->lli_pool
.base
);
669 d40d
->lli_pool
.base
= NULL
;
670 d40d
->lli_pool
.dma_addr
= 0;
678 static void d40_pool_lli_free(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
680 if (d40d
->lli_pool
.dma_addr
)
681 dma_unmap_single(d40c
->base
->dev
, d40d
->lli_pool
.dma_addr
,
682 d40d
->lli_pool
.size
, DMA_TO_DEVICE
);
684 kfree(d40d
->lli_pool
.base
);
685 d40d
->lli_pool
.base
= NULL
;
686 d40d
->lli_pool
.size
= 0;
687 d40d
->lli_log
.src
= NULL
;
688 d40d
->lli_log
.dst
= NULL
;
689 d40d
->lli_phy
.src
= NULL
;
690 d40d
->lli_phy
.dst
= NULL
;
693 static int d40_lcla_alloc_one(struct d40_chan
*d40c
,
694 struct d40_desc
*d40d
)
700 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
703 * Allocate both src and dst at the same time, therefore the half
704 * start on 1 since 0 can't be used since zero is used as end marker.
706 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
707 int idx
= d40c
->phy_chan
->num
* D40_LCLA_LINK_PER_EVENT_GRP
+ i
;
709 if (!d40c
->base
->lcla_pool
.alloc_map
[idx
]) {
710 d40c
->base
->lcla_pool
.alloc_map
[idx
] = d40d
;
717 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
722 static int d40_lcla_free_all(struct d40_chan
*d40c
,
723 struct d40_desc
*d40d
)
729 if (chan_is_physical(d40c
))
732 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
734 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
735 int idx
= d40c
->phy_chan
->num
* D40_LCLA_LINK_PER_EVENT_GRP
+ i
;
737 if (d40c
->base
->lcla_pool
.alloc_map
[idx
] == d40d
) {
738 d40c
->base
->lcla_pool
.alloc_map
[idx
] = NULL
;
740 if (d40d
->lcla_alloc
== 0) {
747 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
753 static void d40_desc_remove(struct d40_desc
*d40d
)
755 list_del(&d40d
->node
);
758 static struct d40_desc
*d40_desc_get(struct d40_chan
*d40c
)
760 struct d40_desc
*desc
= NULL
;
762 if (!list_empty(&d40c
->client
)) {
766 list_for_each_entry_safe(d
, _d
, &d40c
->client
, node
) {
767 if (async_tx_test_ack(&d
->txd
)) {
770 memset(desc
, 0, sizeof(*desc
));
777 desc
= kmem_cache_zalloc(d40c
->base
->desc_slab
, GFP_NOWAIT
);
780 INIT_LIST_HEAD(&desc
->node
);
785 static void d40_desc_free(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
788 d40_pool_lli_free(d40c
, d40d
);
789 d40_lcla_free_all(d40c
, d40d
);
790 kmem_cache_free(d40c
->base
->desc_slab
, d40d
);
793 static void d40_desc_submit(struct d40_chan
*d40c
, struct d40_desc
*desc
)
795 list_add_tail(&desc
->node
, &d40c
->active
);
798 static void d40_phy_lli_load(struct d40_chan
*chan
, struct d40_desc
*desc
)
800 struct d40_phy_lli
*lli_dst
= desc
->lli_phy
.dst
;
801 struct d40_phy_lli
*lli_src
= desc
->lli_phy
.src
;
802 void __iomem
*base
= chan_base(chan
);
804 writel(lli_src
->reg_cfg
, base
+ D40_CHAN_REG_SSCFG
);
805 writel(lli_src
->reg_elt
, base
+ D40_CHAN_REG_SSELT
);
806 writel(lli_src
->reg_ptr
, base
+ D40_CHAN_REG_SSPTR
);
807 writel(lli_src
->reg_lnk
, base
+ D40_CHAN_REG_SSLNK
);
809 writel(lli_dst
->reg_cfg
, base
+ D40_CHAN_REG_SDCFG
);
810 writel(lli_dst
->reg_elt
, base
+ D40_CHAN_REG_SDELT
);
811 writel(lli_dst
->reg_ptr
, base
+ D40_CHAN_REG_SDPTR
);
812 writel(lli_dst
->reg_lnk
, base
+ D40_CHAN_REG_SDLNK
);
815 static void d40_desc_done(struct d40_chan
*d40c
, struct d40_desc
*desc
)
817 list_add_tail(&desc
->node
, &d40c
->done
);
820 static void d40_log_lli_to_lcxa(struct d40_chan
*chan
, struct d40_desc
*desc
)
822 struct d40_lcla_pool
*pool
= &chan
->base
->lcla_pool
;
823 struct d40_log_lli_bidir
*lli
= &desc
->lli_log
;
824 int lli_current
= desc
->lli_current
;
825 int lli_len
= desc
->lli_len
;
826 bool cyclic
= desc
->cyclic
;
827 int curr_lcla
= -EINVAL
;
829 bool use_esram_lcla
= chan
->base
->plat_data
->use_esram_lcla
;
833 * We may have partially running cyclic transfers, in case we did't get
834 * enough LCLA entries.
836 linkback
= cyclic
&& lli_current
== 0;
839 * For linkback, we need one LCLA even with only one link, because we
840 * can't link back to the one in LCPA space
842 if (linkback
|| (lli_len
- lli_current
> 1)) {
844 * If the channel is expected to use only soft_lli don't
845 * allocate a lcla. This is to avoid a HW issue that exists
846 * in some controller during a peripheral to memory transfer
847 * that uses linked lists.
849 if (!(chan
->phy_chan
->use_soft_lli
&&
850 chan
->dma_cfg
.dir
== DMA_DEV_TO_MEM
))
851 curr_lcla
= d40_lcla_alloc_one(chan
, desc
);
853 first_lcla
= curr_lcla
;
857 * For linkback, we normally load the LCPA in the loop since we need to
858 * link it to the second LCLA and not the first. However, if we
859 * couldn't even get a first LCLA, then we have to run in LCPA and
862 if (!linkback
|| curr_lcla
== -EINVAL
) {
863 unsigned int flags
= 0;
865 if (curr_lcla
== -EINVAL
)
866 flags
|= LLI_TERM_INT
;
868 d40_log_lli_lcpa_write(chan
->lcpa
,
869 &lli
->dst
[lli_current
],
870 &lli
->src
[lli_current
],
879 for (; lli_current
< lli_len
; lli_current
++) {
880 unsigned int lcla_offset
= chan
->phy_chan
->num
* 1024 +
882 struct d40_log_lli
*lcla
= pool
->base
+ lcla_offset
;
883 unsigned int flags
= 0;
886 if (lli_current
+ 1 < lli_len
)
887 next_lcla
= d40_lcla_alloc_one(chan
, desc
);
889 next_lcla
= linkback
? first_lcla
: -EINVAL
;
891 if (cyclic
|| next_lcla
== -EINVAL
)
892 flags
|= LLI_TERM_INT
;
894 if (linkback
&& curr_lcla
== first_lcla
) {
895 /* First link goes in both LCPA and LCLA */
896 d40_log_lli_lcpa_write(chan
->lcpa
,
897 &lli
->dst
[lli_current
],
898 &lli
->src
[lli_current
],
903 * One unused LCLA in the cyclic case if the very first
906 d40_log_lli_lcla_write(lcla
,
907 &lli
->dst
[lli_current
],
908 &lli
->src
[lli_current
],
912 * Cache maintenance is not needed if lcla is
915 if (!use_esram_lcla
) {
916 dma_sync_single_range_for_device(chan
->base
->dev
,
917 pool
->dma_addr
, lcla_offset
,
918 2 * sizeof(struct d40_log_lli
),
921 curr_lcla
= next_lcla
;
923 if (curr_lcla
== -EINVAL
|| curr_lcla
== first_lcla
) {
930 desc
->lli_current
= lli_current
;
933 static void d40_desc_load(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
935 if (chan_is_physical(d40c
)) {
936 d40_phy_lli_load(d40c
, d40d
);
937 d40d
->lli_current
= d40d
->lli_len
;
939 d40_log_lli_to_lcxa(d40c
, d40d
);
942 static struct d40_desc
*d40_first_active_get(struct d40_chan
*d40c
)
946 if (list_empty(&d40c
->active
))
949 d
= list_first_entry(&d40c
->active
,
955 /* remove desc from current queue and add it to the pending_queue */
956 static void d40_desc_queue(struct d40_chan
*d40c
, struct d40_desc
*desc
)
958 d40_desc_remove(desc
);
959 desc
->is_in_client_list
= false;
960 list_add_tail(&desc
->node
, &d40c
->pending_queue
);
963 static struct d40_desc
*d40_first_pending(struct d40_chan
*d40c
)
967 if (list_empty(&d40c
->pending_queue
))
970 d
= list_first_entry(&d40c
->pending_queue
,
976 static struct d40_desc
*d40_first_queued(struct d40_chan
*d40c
)
980 if (list_empty(&d40c
->queue
))
983 d
= list_first_entry(&d40c
->queue
,
989 static struct d40_desc
*d40_first_done(struct d40_chan
*d40c
)
991 if (list_empty(&d40c
->done
))
994 return list_first_entry(&d40c
->done
, struct d40_desc
, node
);
997 static int d40_psize_2_burst_size(bool is_log
, int psize
)
1000 if (psize
== STEDMA40_PSIZE_LOG_1
)
1003 if (psize
== STEDMA40_PSIZE_PHY_1
)
1011 * The dma only supports transmitting packages up to
1012 * STEDMA40_MAX_SEG_SIZE * data_width, where data_width is stored in Bytes.
1014 * Calculate the total number of dma elements required to send the entire sg list.
1016 static int d40_size_2_dmalen(int size
, u32 data_width1
, u32 data_width2
)
1019 u32 max_w
= max(data_width1
, data_width2
);
1020 u32 min_w
= min(data_width1
, data_width2
);
1021 u32 seg_max
= ALIGN(STEDMA40_MAX_SEG_SIZE
* min_w
, max_w
);
1023 if (seg_max
> STEDMA40_MAX_SEG_SIZE
)
1026 if (!IS_ALIGNED(size
, max_w
))
1029 if (size
<= seg_max
)
1032 dmalen
= size
/ seg_max
;
1033 if (dmalen
* seg_max
< size
)
1039 static int d40_sg_2_dmalen(struct scatterlist
*sgl
, int sg_len
,
1040 u32 data_width1
, u32 data_width2
)
1042 struct scatterlist
*sg
;
1047 for_each_sg(sgl
, sg
, sg_len
, i
) {
1048 ret
= d40_size_2_dmalen(sg_dma_len(sg
),
1049 data_width1
, data_width2
);
1057 static int __d40_execute_command_phy(struct d40_chan
*d40c
,
1058 enum d40_command command
)
1062 void __iomem
*active_reg
;
1064 unsigned long flags
;
1067 if (command
== D40_DMA_STOP
) {
1068 ret
= __d40_execute_command_phy(d40c
, D40_DMA_SUSPEND_REQ
);
1073 spin_lock_irqsave(&d40c
->base
->execmd_lock
, flags
);
1075 if (d40c
->phy_chan
->num
% 2 == 0)
1076 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
1078 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
1080 if (command
== D40_DMA_SUSPEND_REQ
) {
1081 status
= (readl(active_reg
) &
1082 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1083 D40_CHAN_POS(d40c
->phy_chan
->num
);
1085 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
1089 wmask
= 0xffffffff & ~(D40_CHAN_POS_MASK(d40c
->phy_chan
->num
));
1090 writel(wmask
| (command
<< D40_CHAN_POS(d40c
->phy_chan
->num
)),
1093 if (command
== D40_DMA_SUSPEND_REQ
) {
1095 for (i
= 0 ; i
< D40_SUSPEND_MAX_IT
; i
++) {
1096 status
= (readl(active_reg
) &
1097 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1098 D40_CHAN_POS(d40c
->phy_chan
->num
);
1102 * Reduce the number of bus accesses while
1103 * waiting for the DMA to suspend.
1107 if (status
== D40_DMA_STOP
||
1108 status
== D40_DMA_SUSPENDED
)
1112 if (i
== D40_SUSPEND_MAX_IT
) {
1114 "unable to suspend the chl %d (log: %d) status %x\n",
1115 d40c
->phy_chan
->num
, d40c
->log_num
,
1123 spin_unlock_irqrestore(&d40c
->base
->execmd_lock
, flags
);
1127 static void d40_term_all(struct d40_chan
*d40c
)
1129 struct d40_desc
*d40d
;
1130 struct d40_desc
*_d
;
1132 /* Release completed descriptors */
1133 while ((d40d
= d40_first_done(d40c
))) {
1134 d40_desc_remove(d40d
);
1135 d40_desc_free(d40c
, d40d
);
1138 /* Release active descriptors */
1139 while ((d40d
= d40_first_active_get(d40c
))) {
1140 d40_desc_remove(d40d
);
1141 d40_desc_free(d40c
, d40d
);
1144 /* Release queued descriptors waiting for transfer */
1145 while ((d40d
= d40_first_queued(d40c
))) {
1146 d40_desc_remove(d40d
);
1147 d40_desc_free(d40c
, d40d
);
1150 /* Release pending descriptors */
1151 while ((d40d
= d40_first_pending(d40c
))) {
1152 d40_desc_remove(d40d
);
1153 d40_desc_free(d40c
, d40d
);
1156 /* Release client owned descriptors */
1157 if (!list_empty(&d40c
->client
))
1158 list_for_each_entry_safe(d40d
, _d
, &d40c
->client
, node
) {
1159 d40_desc_remove(d40d
);
1160 d40_desc_free(d40c
, d40d
);
1163 /* Release descriptors in prepare queue */
1164 if (!list_empty(&d40c
->prepare_queue
))
1165 list_for_each_entry_safe(d40d
, _d
,
1166 &d40c
->prepare_queue
, node
) {
1167 d40_desc_remove(d40d
);
1168 d40_desc_free(d40c
, d40d
);
1171 d40c
->pending_tx
= 0;
1174 static void __d40_config_set_event(struct d40_chan
*d40c
,
1175 enum d40_events event_type
, u32 event
,
1178 void __iomem
*addr
= chan_base(d40c
) + reg
;
1182 switch (event_type
) {
1184 case D40_DEACTIVATE_EVENTLINE
:
1186 writel((D40_DEACTIVATE_EVENTLINE
<< D40_EVENTLINE_POS(event
))
1187 | ~D40_EVENTLINE_MASK(event
), addr
);
1190 case D40_SUSPEND_REQ_EVENTLINE
:
1191 status
= (readl(addr
) & D40_EVENTLINE_MASK(event
)) >>
1192 D40_EVENTLINE_POS(event
);
1194 if (status
== D40_DEACTIVATE_EVENTLINE
||
1195 status
== D40_SUSPEND_REQ_EVENTLINE
)
1198 writel((D40_SUSPEND_REQ_EVENTLINE
<< D40_EVENTLINE_POS(event
))
1199 | ~D40_EVENTLINE_MASK(event
), addr
);
1201 for (tries
= 0 ; tries
< D40_SUSPEND_MAX_IT
; tries
++) {
1203 status
= (readl(addr
) & D40_EVENTLINE_MASK(event
)) >>
1204 D40_EVENTLINE_POS(event
);
1208 * Reduce the number of bus accesses while
1209 * waiting for the DMA to suspend.
1213 if (status
== D40_DEACTIVATE_EVENTLINE
)
1217 if (tries
== D40_SUSPEND_MAX_IT
) {
1219 "unable to stop the event_line chl %d (log: %d)"
1220 "status %x\n", d40c
->phy_chan
->num
,
1221 d40c
->log_num
, status
);
1225 case D40_ACTIVATE_EVENTLINE
:
1227 * The hardware sometimes doesn't register the enable when src and dst
1228 * event lines are active on the same logical channel. Retry to ensure
1229 * it does. Usually only one retry is sufficient.
1233 writel((D40_ACTIVATE_EVENTLINE
<<
1234 D40_EVENTLINE_POS(event
)) |
1235 ~D40_EVENTLINE_MASK(event
), addr
);
1237 if (readl(addr
) & D40_EVENTLINE_MASK(event
))
1242 dev_dbg(chan2dev(d40c
),
1243 "[%s] workaround enable S%cLNK (%d tries)\n",
1244 __func__
, reg
== D40_CHAN_REG_SSLNK
? 'S' : 'D',
1250 case D40_ROUND_EVENTLINE
:
1257 static void d40_config_set_event(struct d40_chan
*d40c
,
1258 enum d40_events event_type
)
1260 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
1262 /* Enable event line connected to device (or memcpy) */
1263 if ((d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) ||
1264 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
1265 __d40_config_set_event(d40c
, event_type
, event
,
1266 D40_CHAN_REG_SSLNK
);
1268 if (d40c
->dma_cfg
.dir
!= DMA_DEV_TO_MEM
)
1269 __d40_config_set_event(d40c
, event_type
, event
,
1270 D40_CHAN_REG_SDLNK
);
1273 static u32
d40_chan_has_events(struct d40_chan
*d40c
)
1275 void __iomem
*chanbase
= chan_base(d40c
);
1278 val
= readl(chanbase
+ D40_CHAN_REG_SSLNK
);
1279 val
|= readl(chanbase
+ D40_CHAN_REG_SDLNK
);
1285 __d40_execute_command_log(struct d40_chan
*d40c
, enum d40_command command
)
1287 unsigned long flags
;
1290 void __iomem
*active_reg
;
1292 if (d40c
->phy_chan
->num
% 2 == 0)
1293 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
1295 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
1298 spin_lock_irqsave(&d40c
->phy_chan
->lock
, flags
);
1302 case D40_DMA_SUSPEND_REQ
:
1304 active_status
= (readl(active_reg
) &
1305 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1306 D40_CHAN_POS(d40c
->phy_chan
->num
);
1308 if (active_status
== D40_DMA_RUN
)
1309 d40_config_set_event(d40c
, D40_SUSPEND_REQ_EVENTLINE
);
1311 d40_config_set_event(d40c
, D40_DEACTIVATE_EVENTLINE
);
1313 if (!d40_chan_has_events(d40c
) && (command
== D40_DMA_STOP
))
1314 ret
= __d40_execute_command_phy(d40c
, command
);
1320 d40_config_set_event(d40c
, D40_ACTIVATE_EVENTLINE
);
1321 ret
= __d40_execute_command_phy(d40c
, command
);
1324 case D40_DMA_SUSPENDED
:
1329 spin_unlock_irqrestore(&d40c
->phy_chan
->lock
, flags
);
1333 static int d40_channel_execute_command(struct d40_chan
*d40c
,
1334 enum d40_command command
)
1336 if (chan_is_logical(d40c
))
1337 return __d40_execute_command_log(d40c
, command
);
1339 return __d40_execute_command_phy(d40c
, command
);
1342 static u32
d40_get_prmo(struct d40_chan
*d40c
)
1344 static const unsigned int phy_map
[] = {
1345 [STEDMA40_PCHAN_BASIC_MODE
]
1346 = D40_DREG_PRMO_PCHAN_BASIC
,
1347 [STEDMA40_PCHAN_MODULO_MODE
]
1348 = D40_DREG_PRMO_PCHAN_MODULO
,
1349 [STEDMA40_PCHAN_DOUBLE_DST_MODE
]
1350 = D40_DREG_PRMO_PCHAN_DOUBLE_DST
,
1352 static const unsigned int log_map
[] = {
1353 [STEDMA40_LCHAN_SRC_PHY_DST_LOG
]
1354 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG
,
1355 [STEDMA40_LCHAN_SRC_LOG_DST_PHY
]
1356 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY
,
1357 [STEDMA40_LCHAN_SRC_LOG_DST_LOG
]
1358 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG
,
1361 if (chan_is_physical(d40c
))
1362 return phy_map
[d40c
->dma_cfg
.mode_opt
];
1364 return log_map
[d40c
->dma_cfg
.mode_opt
];
1367 static void d40_config_write(struct d40_chan
*d40c
)
1372 /* Odd addresses are even addresses + 4 */
1373 addr_base
= (d40c
->phy_chan
->num
% 2) * 4;
1374 /* Setup channel mode to logical or physical */
1375 var
= ((u32
)(chan_is_logical(d40c
)) + 1) <<
1376 D40_CHAN_POS(d40c
->phy_chan
->num
);
1377 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMSE
+ addr_base
);
1379 /* Setup operational mode option register */
1380 var
= d40_get_prmo(d40c
) << D40_CHAN_POS(d40c
->phy_chan
->num
);
1382 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMOE
+ addr_base
);
1384 if (chan_is_logical(d40c
)) {
1385 int lidx
= (d40c
->phy_chan
->num
<< D40_SREG_ELEM_LOG_LIDX_POS
)
1386 & D40_SREG_ELEM_LOG_LIDX_MASK
;
1387 void __iomem
*chanbase
= chan_base(d40c
);
1389 /* Set default config for CFG reg */
1390 writel(d40c
->src_def_cfg
, chanbase
+ D40_CHAN_REG_SSCFG
);
1391 writel(d40c
->dst_def_cfg
, chanbase
+ D40_CHAN_REG_SDCFG
);
1393 /* Set LIDX for lcla */
1394 writel(lidx
, chanbase
+ D40_CHAN_REG_SSELT
);
1395 writel(lidx
, chanbase
+ D40_CHAN_REG_SDELT
);
1397 /* Clear LNK which will be used by d40_chan_has_events() */
1398 writel(0, chanbase
+ D40_CHAN_REG_SSLNK
);
1399 writel(0, chanbase
+ D40_CHAN_REG_SDLNK
);
1403 static u32
d40_residue(struct d40_chan
*d40c
)
1407 if (chan_is_logical(d40c
))
1408 num_elt
= (readl(&d40c
->lcpa
->lcsp2
) & D40_MEM_LCSP2_ECNT_MASK
)
1409 >> D40_MEM_LCSP2_ECNT_POS
;
1411 u32 val
= readl(chan_base(d40c
) + D40_CHAN_REG_SDELT
);
1412 num_elt
= (val
& D40_SREG_ELEM_PHY_ECNT_MASK
)
1413 >> D40_SREG_ELEM_PHY_ECNT_POS
;
1416 return num_elt
* d40c
->dma_cfg
.dst_info
.data_width
;
1419 static bool d40_tx_is_linked(struct d40_chan
*d40c
)
1423 if (chan_is_logical(d40c
))
1424 is_link
= readl(&d40c
->lcpa
->lcsp3
) & D40_MEM_LCSP3_DLOS_MASK
;
1426 is_link
= readl(chan_base(d40c
) + D40_CHAN_REG_SDLNK
)
1427 & D40_SREG_LNK_PHYS_LNK_MASK
;
1432 static int d40_pause(struct dma_chan
*chan
)
1434 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
1436 unsigned long flags
;
1438 if (d40c
->phy_chan
== NULL
) {
1439 chan_err(d40c
, "Channel is not allocated!\n");
1446 spin_lock_irqsave(&d40c
->lock
, flags
);
1447 pm_runtime_get_sync(d40c
->base
->dev
);
1449 res
= d40_channel_execute_command(d40c
, D40_DMA_SUSPEND_REQ
);
1451 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1452 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1453 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1457 static int d40_resume(struct dma_chan
*chan
)
1459 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
1461 unsigned long flags
;
1463 if (d40c
->phy_chan
== NULL
) {
1464 chan_err(d40c
, "Channel is not allocated!\n");
1471 spin_lock_irqsave(&d40c
->lock
, flags
);
1472 pm_runtime_get_sync(d40c
->base
->dev
);
1474 /* If bytes left to transfer or linked tx resume job */
1475 if (d40_residue(d40c
) || d40_tx_is_linked(d40c
))
1476 res
= d40_channel_execute_command(d40c
, D40_DMA_RUN
);
1478 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1479 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1480 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1484 static dma_cookie_t
d40_tx_submit(struct dma_async_tx_descriptor
*tx
)
1486 struct d40_chan
*d40c
= container_of(tx
->chan
,
1489 struct d40_desc
*d40d
= container_of(tx
, struct d40_desc
, txd
);
1490 unsigned long flags
;
1491 dma_cookie_t cookie
;
1493 spin_lock_irqsave(&d40c
->lock
, flags
);
1494 cookie
= dma_cookie_assign(tx
);
1495 d40_desc_queue(d40c
, d40d
);
1496 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1501 static int d40_start(struct d40_chan
*d40c
)
1503 return d40_channel_execute_command(d40c
, D40_DMA_RUN
);
1506 static struct d40_desc
*d40_queue_start(struct d40_chan
*d40c
)
1508 struct d40_desc
*d40d
;
1511 /* Start queued jobs, if any */
1512 d40d
= d40_first_queued(d40c
);
1517 pm_runtime_get_sync(d40c
->base
->dev
);
1520 /* Remove from queue */
1521 d40_desc_remove(d40d
);
1523 /* Add to active queue */
1524 d40_desc_submit(d40c
, d40d
);
1526 /* Initiate DMA job */
1527 d40_desc_load(d40c
, d40d
);
1530 err
= d40_start(d40c
);
1539 /* called from interrupt context */
1540 static void dma_tc_handle(struct d40_chan
*d40c
)
1542 struct d40_desc
*d40d
;
1544 /* Get first active entry from list */
1545 d40d
= d40_first_active_get(d40c
);
1552 * If this was a paritially loaded list, we need to reloaded
1553 * it, and only when the list is completed. We need to check
1554 * for done because the interrupt will hit for every link, and
1555 * not just the last one.
1557 if (d40d
->lli_current
< d40d
->lli_len
1558 && !d40_tx_is_linked(d40c
)
1559 && !d40_residue(d40c
)) {
1560 d40_lcla_free_all(d40c
, d40d
);
1561 d40_desc_load(d40c
, d40d
);
1562 (void) d40_start(d40c
);
1564 if (d40d
->lli_current
== d40d
->lli_len
)
1565 d40d
->lli_current
= 0;
1568 d40_lcla_free_all(d40c
, d40d
);
1570 if (d40d
->lli_current
< d40d
->lli_len
) {
1571 d40_desc_load(d40c
, d40d
);
1573 (void) d40_start(d40c
);
1577 if (d40_queue_start(d40c
) == NULL
) {
1580 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1581 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1584 d40_desc_remove(d40d
);
1585 d40_desc_done(d40c
, d40d
);
1589 tasklet_schedule(&d40c
->tasklet
);
1593 static void dma_tasklet(unsigned long data
)
1595 struct d40_chan
*d40c
= (struct d40_chan
*) data
;
1596 struct d40_desc
*d40d
;
1597 unsigned long flags
;
1598 bool callback_active
;
1599 dma_async_tx_callback callback
;
1600 void *callback_param
;
1602 spin_lock_irqsave(&d40c
->lock
, flags
);
1604 /* Get first entry from the done list */
1605 d40d
= d40_first_done(d40c
);
1607 /* Check if we have reached here for cyclic job */
1608 d40d
= d40_first_active_get(d40c
);
1609 if (d40d
== NULL
|| !d40d
->cyclic
)
1614 dma_cookie_complete(&d40d
->txd
);
1617 * If terminating a channel pending_tx is set to zero.
1618 * This prevents any finished active jobs to return to the client.
1620 if (d40c
->pending_tx
== 0) {
1621 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1625 /* Callback to client */
1626 callback_active
= !!(d40d
->txd
.flags
& DMA_PREP_INTERRUPT
);
1627 callback
= d40d
->txd
.callback
;
1628 callback_param
= d40d
->txd
.callback_param
;
1630 if (!d40d
->cyclic
) {
1631 if (async_tx_test_ack(&d40d
->txd
)) {
1632 d40_desc_remove(d40d
);
1633 d40_desc_free(d40c
, d40d
);
1634 } else if (!d40d
->is_in_client_list
) {
1635 d40_desc_remove(d40d
);
1636 d40_lcla_free_all(d40c
, d40d
);
1637 list_add_tail(&d40d
->node
, &d40c
->client
);
1638 d40d
->is_in_client_list
= true;
1644 if (d40c
->pending_tx
)
1645 tasklet_schedule(&d40c
->tasklet
);
1647 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1649 if (callback_active
&& callback
)
1650 callback(callback_param
);
1655 /* Rescue manouver if receiving double interrupts */
1656 if (d40c
->pending_tx
> 0)
1658 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1661 static irqreturn_t
d40_handle_interrupt(int irq
, void *data
)
1667 struct d40_chan
*d40c
;
1668 unsigned long flags
;
1669 struct d40_base
*base
= data
;
1670 u32 regs
[base
->gen_dmac
.il_size
];
1671 struct d40_interrupt_lookup
*il
= base
->gen_dmac
.il
;
1672 u32 il_size
= base
->gen_dmac
.il_size
;
1674 spin_lock_irqsave(&base
->interrupt_lock
, flags
);
1676 /* Read interrupt status of both logical and physical channels */
1677 for (i
= 0; i
< il_size
; i
++)
1678 regs
[i
] = readl(base
->virtbase
+ il
[i
].src
);
1682 chan
= find_next_bit((unsigned long *)regs
,
1683 BITS_PER_LONG
* il_size
, chan
+ 1);
1685 /* No more set bits found? */
1686 if (chan
== BITS_PER_LONG
* il_size
)
1689 row
= chan
/ BITS_PER_LONG
;
1690 idx
= chan
& (BITS_PER_LONG
- 1);
1692 if (il
[row
].offset
== D40_PHY_CHAN
)
1693 d40c
= base
->lookup_phy_chans
[idx
];
1695 d40c
= base
->lookup_log_chans
[il
[row
].offset
+ idx
];
1699 * No error because this can happen if something else
1700 * in the system is using the channel.
1706 writel(BIT(idx
), base
->virtbase
+ il
[row
].clr
);
1708 spin_lock(&d40c
->lock
);
1710 if (!il
[row
].is_error
)
1711 dma_tc_handle(d40c
);
1713 d40_err(base
->dev
, "IRQ chan: %ld offset %d idx %d\n",
1714 chan
, il
[row
].offset
, idx
);
1716 spin_unlock(&d40c
->lock
);
1719 spin_unlock_irqrestore(&base
->interrupt_lock
, flags
);
1724 static int d40_validate_conf(struct d40_chan
*d40c
,
1725 struct stedma40_chan_cfg
*conf
)
1728 bool is_log
= conf
->mode
== STEDMA40_MODE_LOGICAL
;
1731 chan_err(d40c
, "Invalid direction.\n");
1735 if ((is_log
&& conf
->dev_type
> d40c
->base
->num_log_chans
) ||
1736 (!is_log
&& conf
->dev_type
> d40c
->base
->num_phy_chans
) ||
1737 (conf
->dev_type
< 0)) {
1738 chan_err(d40c
, "Invalid device type (%d)\n", conf
->dev_type
);
1742 if (conf
->dir
== DMA_DEV_TO_DEV
) {
1744 * DMAC HW supports it. Will be added to this driver,
1745 * in case any dma client requires it.
1747 chan_err(d40c
, "periph to periph not supported\n");
1751 if (d40_psize_2_burst_size(is_log
, conf
->src_info
.psize
) *
1752 conf
->src_info
.data_width
!=
1753 d40_psize_2_burst_size(is_log
, conf
->dst_info
.psize
) *
1754 conf
->dst_info
.data_width
) {
1756 * The DMAC hardware only supports
1757 * src (burst x width) == dst (burst x width)
1760 chan_err(d40c
, "src (burst x width) != dst (burst x width)\n");
1767 static bool d40_alloc_mask_set(struct d40_phy_res
*phy
,
1768 bool is_src
, int log_event_line
, bool is_log
,
1771 unsigned long flags
;
1772 spin_lock_irqsave(&phy
->lock
, flags
);
1774 *first_user
= ((phy
->allocated_src
| phy
->allocated_dst
)
1778 /* Physical interrupts are masked per physical full channel */
1779 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
1780 phy
->allocated_dst
== D40_ALLOC_FREE
) {
1781 phy
->allocated_dst
= D40_ALLOC_PHY
;
1782 phy
->allocated_src
= D40_ALLOC_PHY
;
1788 /* Logical channel */
1790 if (phy
->allocated_src
== D40_ALLOC_PHY
)
1793 if (phy
->allocated_src
== D40_ALLOC_FREE
)
1794 phy
->allocated_src
= D40_ALLOC_LOG_FREE
;
1796 if (!(phy
->allocated_src
& BIT(log_event_line
))) {
1797 phy
->allocated_src
|= BIT(log_event_line
);
1802 if (phy
->allocated_dst
== D40_ALLOC_PHY
)
1805 if (phy
->allocated_dst
== D40_ALLOC_FREE
)
1806 phy
->allocated_dst
= D40_ALLOC_LOG_FREE
;
1808 if (!(phy
->allocated_dst
& BIT(log_event_line
))) {
1809 phy
->allocated_dst
|= BIT(log_event_line
);
1816 spin_unlock_irqrestore(&phy
->lock
, flags
);
1819 spin_unlock_irqrestore(&phy
->lock
, flags
);
1823 static bool d40_alloc_mask_free(struct d40_phy_res
*phy
, bool is_src
,
1826 unsigned long flags
;
1827 bool is_free
= false;
1829 spin_lock_irqsave(&phy
->lock
, flags
);
1830 if (!log_event_line
) {
1831 phy
->allocated_dst
= D40_ALLOC_FREE
;
1832 phy
->allocated_src
= D40_ALLOC_FREE
;
1837 /* Logical channel */
1839 phy
->allocated_src
&= ~BIT(log_event_line
);
1840 if (phy
->allocated_src
== D40_ALLOC_LOG_FREE
)
1841 phy
->allocated_src
= D40_ALLOC_FREE
;
1843 phy
->allocated_dst
&= ~BIT(log_event_line
);
1844 if (phy
->allocated_dst
== D40_ALLOC_LOG_FREE
)
1845 phy
->allocated_dst
= D40_ALLOC_FREE
;
1848 is_free
= ((phy
->allocated_src
| phy
->allocated_dst
) ==
1852 spin_unlock_irqrestore(&phy
->lock
, flags
);
1857 static int d40_allocate_channel(struct d40_chan
*d40c
, bool *first_phy_user
)
1859 int dev_type
= d40c
->dma_cfg
.dev_type
;
1862 struct d40_phy_res
*phys
;
1868 bool is_log
= d40c
->dma_cfg
.mode
== STEDMA40_MODE_LOGICAL
;
1870 phys
= d40c
->base
->phy_res
;
1871 num_phy_chans
= d40c
->base
->num_phy_chans
;
1873 if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) {
1874 log_num
= 2 * dev_type
;
1876 } else if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
1877 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
1878 /* dst event lines are used for logical memcpy */
1879 log_num
= 2 * dev_type
+ 1;
1884 event_group
= D40_TYPE_TO_GROUP(dev_type
);
1885 event_line
= D40_TYPE_TO_EVENT(dev_type
);
1888 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
1889 /* Find physical half channel */
1890 if (d40c
->dma_cfg
.use_fixed_channel
) {
1891 i
= d40c
->dma_cfg
.phy_channel
;
1892 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1897 for (i
= 0; i
< num_phy_chans
; i
++) {
1898 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1905 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1906 int phy_num
= j
+ event_group
* 2;
1907 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1908 if (d40_alloc_mask_set(&phys
[i
],
1918 d40c
->phy_chan
= &phys
[i
];
1919 d40c
->log_num
= D40_PHY_CHAN
;
1925 /* Find logical channel */
1926 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1927 int phy_num
= j
+ event_group
* 2;
1929 if (d40c
->dma_cfg
.use_fixed_channel
) {
1930 i
= d40c
->dma_cfg
.phy_channel
;
1932 if ((i
!= phy_num
) && (i
!= phy_num
+ 1)) {
1933 dev_err(chan2dev(d40c
),
1934 "invalid fixed phy channel %d\n", i
);
1938 if (d40_alloc_mask_set(&phys
[i
], is_src
, event_line
,
1939 is_log
, first_phy_user
))
1942 dev_err(chan2dev(d40c
),
1943 "could not allocate fixed phy channel %d\n", i
);
1948 * Spread logical channels across all available physical rather
1949 * than pack every logical channel at the first available phy
1953 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1954 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1960 for (i
= phy_num
+ 1; i
>= phy_num
; i
--) {
1961 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1971 d40c
->phy_chan
= &phys
[i
];
1972 d40c
->log_num
= log_num
;
1976 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = d40c
;
1978 d40c
->base
->lookup_phy_chans
[d40c
->phy_chan
->num
] = d40c
;
1984 static int d40_config_memcpy(struct d40_chan
*d40c
)
1986 dma_cap_mask_t cap
= d40c
->chan
.device
->cap_mask
;
1988 if (dma_has_cap(DMA_MEMCPY
, cap
) && !dma_has_cap(DMA_SLAVE
, cap
)) {
1989 d40c
->dma_cfg
= dma40_memcpy_conf_log
;
1990 d40c
->dma_cfg
.dev_type
= dma40_memcpy_channels
[d40c
->chan
.chan_id
];
1992 d40_log_cfg(&d40c
->dma_cfg
,
1993 &d40c
->log_def
.lcsp1
, &d40c
->log_def
.lcsp3
);
1995 } else if (dma_has_cap(DMA_MEMCPY
, cap
) &&
1996 dma_has_cap(DMA_SLAVE
, cap
)) {
1997 d40c
->dma_cfg
= dma40_memcpy_conf_phy
;
1999 /* Generate interrrupt at end of transfer or relink. */
2000 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_TIM_POS
);
2002 /* Generate interrupt on error. */
2003 d40c
->src_def_cfg
|= BIT(D40_SREG_CFG_EIM_POS
);
2004 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_EIM_POS
);
2007 chan_err(d40c
, "No memcpy\n");
2014 static int d40_free_dma(struct d40_chan
*d40c
)
2018 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
2019 struct d40_phy_res
*phy
= d40c
->phy_chan
;
2022 /* Terminate all queued and active transfers */
2026 chan_err(d40c
, "phy == null\n");
2030 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
2031 phy
->allocated_dst
== D40_ALLOC_FREE
) {
2032 chan_err(d40c
, "channel already free\n");
2036 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
2037 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
)
2039 else if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
)
2042 chan_err(d40c
, "Unknown direction\n");
2046 pm_runtime_get_sync(d40c
->base
->dev
);
2047 res
= d40_channel_execute_command(d40c
, D40_DMA_STOP
);
2049 chan_err(d40c
, "stop failed\n");
2053 d40_alloc_mask_free(phy
, is_src
, chan_is_logical(d40c
) ? event
: 0);
2055 if (chan_is_logical(d40c
))
2056 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = NULL
;
2058 d40c
->base
->lookup_phy_chans
[phy
->num
] = NULL
;
2061 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2062 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2066 d40c
->phy_chan
= NULL
;
2067 d40c
->configured
= false;
2070 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2071 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2075 static bool d40_is_paused(struct d40_chan
*d40c
)
2077 void __iomem
*chanbase
= chan_base(d40c
);
2078 bool is_paused
= false;
2079 unsigned long flags
;
2080 void __iomem
*active_reg
;
2082 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
2084 spin_lock_irqsave(&d40c
->lock
, flags
);
2086 if (chan_is_physical(d40c
)) {
2087 if (d40c
->phy_chan
->num
% 2 == 0)
2088 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
2090 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
2092 status
= (readl(active_reg
) &
2093 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
2094 D40_CHAN_POS(d40c
->phy_chan
->num
);
2095 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
2101 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
2102 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
2103 status
= readl(chanbase
+ D40_CHAN_REG_SDLNK
);
2104 } else if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) {
2105 status
= readl(chanbase
+ D40_CHAN_REG_SSLNK
);
2107 chan_err(d40c
, "Unknown direction\n");
2111 status
= (status
& D40_EVENTLINE_MASK(event
)) >>
2112 D40_EVENTLINE_POS(event
);
2114 if (status
!= D40_DMA_RUN
)
2117 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2122 static u32
stedma40_residue(struct dma_chan
*chan
)
2124 struct d40_chan
*d40c
=
2125 container_of(chan
, struct d40_chan
, chan
);
2127 unsigned long flags
;
2129 spin_lock_irqsave(&d40c
->lock
, flags
);
2130 bytes_left
= d40_residue(d40c
);
2131 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2137 d40_prep_sg_log(struct d40_chan
*chan
, struct d40_desc
*desc
,
2138 struct scatterlist
*sg_src
, struct scatterlist
*sg_dst
,
2139 unsigned int sg_len
, dma_addr_t src_dev_addr
,
2140 dma_addr_t dst_dev_addr
)
2142 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2143 struct stedma40_half_channel_info
*src_info
= &cfg
->src_info
;
2144 struct stedma40_half_channel_info
*dst_info
= &cfg
->dst_info
;
2147 ret
= d40_log_sg_to_lli(sg_src
, sg_len
,
2150 chan
->log_def
.lcsp1
,
2151 src_info
->data_width
,
2152 dst_info
->data_width
);
2154 ret
= d40_log_sg_to_lli(sg_dst
, sg_len
,
2157 chan
->log_def
.lcsp3
,
2158 dst_info
->data_width
,
2159 src_info
->data_width
);
2161 return ret
< 0 ? ret
: 0;
2165 d40_prep_sg_phy(struct d40_chan
*chan
, struct d40_desc
*desc
,
2166 struct scatterlist
*sg_src
, struct scatterlist
*sg_dst
,
2167 unsigned int sg_len
, dma_addr_t src_dev_addr
,
2168 dma_addr_t dst_dev_addr
)
2170 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2171 struct stedma40_half_channel_info
*src_info
= &cfg
->src_info
;
2172 struct stedma40_half_channel_info
*dst_info
= &cfg
->dst_info
;
2173 unsigned long flags
= 0;
2177 flags
|= LLI_CYCLIC
| LLI_TERM_INT
;
2179 ret
= d40_phy_sg_to_lli(sg_src
, sg_len
, src_dev_addr
,
2181 virt_to_phys(desc
->lli_phy
.src
),
2183 src_info
, dst_info
, flags
);
2185 ret
= d40_phy_sg_to_lli(sg_dst
, sg_len
, dst_dev_addr
,
2187 virt_to_phys(desc
->lli_phy
.dst
),
2189 dst_info
, src_info
, flags
);
2191 dma_sync_single_for_device(chan
->base
->dev
, desc
->lli_pool
.dma_addr
,
2192 desc
->lli_pool
.size
, DMA_TO_DEVICE
);
2194 return ret
< 0 ? ret
: 0;
2197 static struct d40_desc
*
2198 d40_prep_desc(struct d40_chan
*chan
, struct scatterlist
*sg
,
2199 unsigned int sg_len
, unsigned long dma_flags
)
2201 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2202 struct d40_desc
*desc
;
2205 desc
= d40_desc_get(chan
);
2209 desc
->lli_len
= d40_sg_2_dmalen(sg
, sg_len
, cfg
->src_info
.data_width
,
2210 cfg
->dst_info
.data_width
);
2211 if (desc
->lli_len
< 0) {
2212 chan_err(chan
, "Unaligned size\n");
2216 ret
= d40_pool_lli_alloc(chan
, desc
, desc
->lli_len
);
2218 chan_err(chan
, "Could not allocate lli\n");
2222 desc
->lli_current
= 0;
2223 desc
->txd
.flags
= dma_flags
;
2224 desc
->txd
.tx_submit
= d40_tx_submit
;
2226 dma_async_tx_descriptor_init(&desc
->txd
, &chan
->chan
);
2231 d40_desc_free(chan
, desc
);
2235 static struct dma_async_tx_descriptor
*
2236 d40_prep_sg(struct dma_chan
*dchan
, struct scatterlist
*sg_src
,
2237 struct scatterlist
*sg_dst
, unsigned int sg_len
,
2238 enum dma_transfer_direction direction
, unsigned long dma_flags
)
2240 struct d40_chan
*chan
= container_of(dchan
, struct d40_chan
, chan
);
2241 dma_addr_t src_dev_addr
= 0;
2242 dma_addr_t dst_dev_addr
= 0;
2243 struct d40_desc
*desc
;
2244 unsigned long flags
;
2247 if (!chan
->phy_chan
) {
2248 chan_err(chan
, "Cannot prepare unallocated channel\n");
2252 spin_lock_irqsave(&chan
->lock
, flags
);
2254 desc
= d40_prep_desc(chan
, sg_src
, sg_len
, dma_flags
);
2258 if (sg_next(&sg_src
[sg_len
- 1]) == sg_src
)
2259 desc
->cyclic
= true;
2261 if (direction
== DMA_DEV_TO_MEM
)
2262 src_dev_addr
= chan
->runtime_addr
;
2263 else if (direction
== DMA_MEM_TO_DEV
)
2264 dst_dev_addr
= chan
->runtime_addr
;
2266 if (chan_is_logical(chan
))
2267 ret
= d40_prep_sg_log(chan
, desc
, sg_src
, sg_dst
,
2268 sg_len
, src_dev_addr
, dst_dev_addr
);
2270 ret
= d40_prep_sg_phy(chan
, desc
, sg_src
, sg_dst
,
2271 sg_len
, src_dev_addr
, dst_dev_addr
);
2274 chan_err(chan
, "Failed to prepare %s sg job: %d\n",
2275 chan_is_logical(chan
) ? "log" : "phy", ret
);
2280 * add descriptor to the prepare queue in order to be able
2281 * to free them later in terminate_all
2283 list_add_tail(&desc
->node
, &chan
->prepare_queue
);
2285 spin_unlock_irqrestore(&chan
->lock
, flags
);
2291 d40_desc_free(chan
, desc
);
2292 spin_unlock_irqrestore(&chan
->lock
, flags
);
2296 bool stedma40_filter(struct dma_chan
*chan
, void *data
)
2298 struct stedma40_chan_cfg
*info
= data
;
2299 struct d40_chan
*d40c
=
2300 container_of(chan
, struct d40_chan
, chan
);
2304 err
= d40_validate_conf(d40c
, info
);
2306 d40c
->dma_cfg
= *info
;
2308 err
= d40_config_memcpy(d40c
);
2311 d40c
->configured
= true;
2315 EXPORT_SYMBOL(stedma40_filter
);
2317 static void __d40_set_prio_rt(struct d40_chan
*d40c
, int dev_type
, bool src
)
2319 bool realtime
= d40c
->dma_cfg
.realtime
;
2320 bool highprio
= d40c
->dma_cfg
.high_priority
;
2322 u32 event
= D40_TYPE_TO_EVENT(dev_type
);
2323 u32 group
= D40_TYPE_TO_GROUP(dev_type
);
2324 u32 bit
= BIT(event
);
2326 struct d40_gen_dmac
*dmac
= &d40c
->base
->gen_dmac
;
2328 rtreg
= realtime
? dmac
->realtime_en
: dmac
->realtime_clear
;
2330 * Due to a hardware bug, in some cases a logical channel triggered by
2331 * a high priority destination event line can generate extra packet
2334 * The workaround is to not set the high priority level for the
2335 * destination event lines that trigger logical channels.
2337 if (!src
&& chan_is_logical(d40c
))
2340 prioreg
= highprio
? dmac
->high_prio_en
: dmac
->high_prio_clear
;
2342 /* Destination event lines are stored in the upper halfword */
2346 writel(bit
, d40c
->base
->virtbase
+ prioreg
+ group
* 4);
2347 writel(bit
, d40c
->base
->virtbase
+ rtreg
+ group
* 4);
2350 static void d40_set_prio_realtime(struct d40_chan
*d40c
)
2352 if (d40c
->base
->rev
< 3)
2355 if ((d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) ||
2356 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
2357 __d40_set_prio_rt(d40c
, d40c
->dma_cfg
.dev_type
, true);
2359 if ((d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
) ||
2360 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
2361 __d40_set_prio_rt(d40c
, d40c
->dma_cfg
.dev_type
, false);
2364 #define D40_DT_FLAGS_MODE(flags) ((flags >> 0) & 0x1)
2365 #define D40_DT_FLAGS_DIR(flags) ((flags >> 1) & 0x1)
2366 #define D40_DT_FLAGS_BIG_ENDIAN(flags) ((flags >> 2) & 0x1)
2367 #define D40_DT_FLAGS_FIXED_CHAN(flags) ((flags >> 3) & 0x1)
2368 #define D40_DT_FLAGS_HIGH_PRIO(flags) ((flags >> 4) & 0x1)
2370 static struct dma_chan
*d40_xlate(struct of_phandle_args
*dma_spec
,
2371 struct of_dma
*ofdma
)
2373 struct stedma40_chan_cfg cfg
;
2377 memset(&cfg
, 0, sizeof(struct stedma40_chan_cfg
));
2380 dma_cap_set(DMA_SLAVE
, cap
);
2382 cfg
.dev_type
= dma_spec
->args
[0];
2383 flags
= dma_spec
->args
[2];
2385 switch (D40_DT_FLAGS_MODE(flags
)) {
2386 case 0: cfg
.mode
= STEDMA40_MODE_LOGICAL
; break;
2387 case 1: cfg
.mode
= STEDMA40_MODE_PHYSICAL
; break;
2390 switch (D40_DT_FLAGS_DIR(flags
)) {
2392 cfg
.dir
= DMA_MEM_TO_DEV
;
2393 cfg
.dst_info
.big_endian
= D40_DT_FLAGS_BIG_ENDIAN(flags
);
2396 cfg
.dir
= DMA_DEV_TO_MEM
;
2397 cfg
.src_info
.big_endian
= D40_DT_FLAGS_BIG_ENDIAN(flags
);
2401 if (D40_DT_FLAGS_FIXED_CHAN(flags
)) {
2402 cfg
.phy_channel
= dma_spec
->args
[1];
2403 cfg
.use_fixed_channel
= true;
2406 if (D40_DT_FLAGS_HIGH_PRIO(flags
))
2407 cfg
.high_priority
= true;
2409 return dma_request_channel(cap
, stedma40_filter
, &cfg
);
2412 /* DMA ENGINE functions */
2413 static int d40_alloc_chan_resources(struct dma_chan
*chan
)
2416 unsigned long flags
;
2417 struct d40_chan
*d40c
=
2418 container_of(chan
, struct d40_chan
, chan
);
2420 spin_lock_irqsave(&d40c
->lock
, flags
);
2422 dma_cookie_init(chan
);
2424 /* If no dma configuration is set use default configuration (memcpy) */
2425 if (!d40c
->configured
) {
2426 err
= d40_config_memcpy(d40c
);
2428 chan_err(d40c
, "Failed to configure memcpy channel\n");
2433 err
= d40_allocate_channel(d40c
, &is_free_phy
);
2435 chan_err(d40c
, "Failed to allocate channel\n");
2436 d40c
->configured
= false;
2440 pm_runtime_get_sync(d40c
->base
->dev
);
2442 d40_set_prio_realtime(d40c
);
2444 if (chan_is_logical(d40c
)) {
2445 if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
)
2446 d40c
->lcpa
= d40c
->base
->lcpa_base
+
2447 d40c
->dma_cfg
.dev_type
* D40_LCPA_CHAN_SIZE
;
2449 d40c
->lcpa
= d40c
->base
->lcpa_base
+
2450 d40c
->dma_cfg
.dev_type
*
2451 D40_LCPA_CHAN_SIZE
+ D40_LCPA_CHAN_DST_DELTA
;
2453 /* Unmask the Global Interrupt Mask. */
2454 d40c
->src_def_cfg
|= BIT(D40_SREG_CFG_LOG_GIM_POS
);
2455 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_LOG_GIM_POS
);
2458 dev_dbg(chan2dev(d40c
), "allocated %s channel (phy %d%s)\n",
2459 chan_is_logical(d40c
) ? "logical" : "physical",
2460 d40c
->phy_chan
->num
,
2461 d40c
->dma_cfg
.use_fixed_channel
? ", fixed" : "");
2465 * Only write channel configuration to the DMA if the physical
2466 * resource is free. In case of multiple logical channels
2467 * on the same physical resource, only the first write is necessary.
2470 d40_config_write(d40c
);
2472 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2473 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2474 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2478 static void d40_free_chan_resources(struct dma_chan
*chan
)
2480 struct d40_chan
*d40c
=
2481 container_of(chan
, struct d40_chan
, chan
);
2483 unsigned long flags
;
2485 if (d40c
->phy_chan
== NULL
) {
2486 chan_err(d40c
, "Cannot free unallocated channel\n");
2490 spin_lock_irqsave(&d40c
->lock
, flags
);
2492 err
= d40_free_dma(d40c
);
2495 chan_err(d40c
, "Failed to free channel\n");
2496 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2499 static struct dma_async_tx_descriptor
*d40_prep_memcpy(struct dma_chan
*chan
,
2503 unsigned long dma_flags
)
2505 struct scatterlist dst_sg
;
2506 struct scatterlist src_sg
;
2508 sg_init_table(&dst_sg
, 1);
2509 sg_init_table(&src_sg
, 1);
2511 sg_dma_address(&dst_sg
) = dst
;
2512 sg_dma_address(&src_sg
) = src
;
2514 sg_dma_len(&dst_sg
) = size
;
2515 sg_dma_len(&src_sg
) = size
;
2517 return d40_prep_sg(chan
, &src_sg
, &dst_sg
, 1,
2518 DMA_MEM_TO_MEM
, dma_flags
);
2521 static struct dma_async_tx_descriptor
*
2522 d40_prep_memcpy_sg(struct dma_chan
*chan
,
2523 struct scatterlist
*dst_sg
, unsigned int dst_nents
,
2524 struct scatterlist
*src_sg
, unsigned int src_nents
,
2525 unsigned long dma_flags
)
2527 if (dst_nents
!= src_nents
)
2530 return d40_prep_sg(chan
, src_sg
, dst_sg
, src_nents
,
2531 DMA_MEM_TO_MEM
, dma_flags
);
2534 static struct dma_async_tx_descriptor
*
2535 d40_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
2536 unsigned int sg_len
, enum dma_transfer_direction direction
,
2537 unsigned long dma_flags
, void *context
)
2539 if (!is_slave_direction(direction
))
2542 return d40_prep_sg(chan
, sgl
, sgl
, sg_len
, direction
, dma_flags
);
2545 static struct dma_async_tx_descriptor
*
2546 dma40_prep_dma_cyclic(struct dma_chan
*chan
, dma_addr_t dma_addr
,
2547 size_t buf_len
, size_t period_len
,
2548 enum dma_transfer_direction direction
, unsigned long flags
)
2550 unsigned int periods
= buf_len
/ period_len
;
2551 struct dma_async_tx_descriptor
*txd
;
2552 struct scatterlist
*sg
;
2555 sg
= kcalloc(periods
+ 1, sizeof(struct scatterlist
), GFP_NOWAIT
);
2559 for (i
= 0; i
< periods
; i
++) {
2560 sg_dma_address(&sg
[i
]) = dma_addr
;
2561 sg_dma_len(&sg
[i
]) = period_len
;
2562 dma_addr
+= period_len
;
2565 sg
[periods
].offset
= 0;
2566 sg_dma_len(&sg
[periods
]) = 0;
2567 sg
[periods
].page_link
=
2568 ((unsigned long)sg
| 0x01) & ~0x02;
2570 txd
= d40_prep_sg(chan
, sg
, sg
, periods
, direction
,
2571 DMA_PREP_INTERRUPT
);
2578 static enum dma_status
d40_tx_status(struct dma_chan
*chan
,
2579 dma_cookie_t cookie
,
2580 struct dma_tx_state
*txstate
)
2582 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2583 enum dma_status ret
;
2585 if (d40c
->phy_chan
== NULL
) {
2586 chan_err(d40c
, "Cannot read status of unallocated channel\n");
2590 ret
= dma_cookie_status(chan
, cookie
, txstate
);
2591 if (ret
!= DMA_COMPLETE
&& txstate
)
2592 dma_set_residue(txstate
, stedma40_residue(chan
));
2594 if (d40_is_paused(d40c
))
2600 static void d40_issue_pending(struct dma_chan
*chan
)
2602 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2603 unsigned long flags
;
2605 if (d40c
->phy_chan
== NULL
) {
2606 chan_err(d40c
, "Channel is not allocated!\n");
2610 spin_lock_irqsave(&d40c
->lock
, flags
);
2612 list_splice_tail_init(&d40c
->pending_queue
, &d40c
->queue
);
2614 /* Busy means that queued jobs are already being processed */
2616 (void) d40_queue_start(d40c
);
2618 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2621 static int d40_terminate_all(struct dma_chan
*chan
)
2623 unsigned long flags
;
2624 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2627 if (d40c
->phy_chan
== NULL
) {
2628 chan_err(d40c
, "Channel is not allocated!\n");
2632 spin_lock_irqsave(&d40c
->lock
, flags
);
2634 pm_runtime_get_sync(d40c
->base
->dev
);
2635 ret
= d40_channel_execute_command(d40c
, D40_DMA_STOP
);
2637 chan_err(d40c
, "Failed to stop channel\n");
2640 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2641 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2643 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2644 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2648 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2653 dma40_config_to_halfchannel(struct d40_chan
*d40c
,
2654 struct stedma40_half_channel_info
*info
,
2659 if (chan_is_logical(d40c
)) {
2661 psize
= STEDMA40_PSIZE_LOG_16
;
2662 else if (maxburst
>= 8)
2663 psize
= STEDMA40_PSIZE_LOG_8
;
2664 else if (maxburst
>= 4)
2665 psize
= STEDMA40_PSIZE_LOG_4
;
2667 psize
= STEDMA40_PSIZE_LOG_1
;
2670 psize
= STEDMA40_PSIZE_PHY_16
;
2671 else if (maxburst
>= 8)
2672 psize
= STEDMA40_PSIZE_PHY_8
;
2673 else if (maxburst
>= 4)
2674 psize
= STEDMA40_PSIZE_PHY_4
;
2676 psize
= STEDMA40_PSIZE_PHY_1
;
2679 info
->psize
= psize
;
2680 info
->flow_ctrl
= STEDMA40_NO_FLOW_CTRL
;
2685 /* Runtime reconfiguration extension */
2686 static int d40_set_runtime_config(struct dma_chan
*chan
,
2687 struct dma_slave_config
*config
)
2689 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2690 struct stedma40_chan_cfg
*cfg
= &d40c
->dma_cfg
;
2691 enum dma_slave_buswidth src_addr_width
, dst_addr_width
;
2692 dma_addr_t config_addr
;
2693 u32 src_maxburst
, dst_maxburst
;
2696 if (d40c
->phy_chan
== NULL
) {
2697 chan_err(d40c
, "Channel is not allocated!\n");
2701 src_addr_width
= config
->src_addr_width
;
2702 src_maxburst
= config
->src_maxburst
;
2703 dst_addr_width
= config
->dst_addr_width
;
2704 dst_maxburst
= config
->dst_maxburst
;
2706 if (config
->direction
== DMA_DEV_TO_MEM
) {
2707 config_addr
= config
->src_addr
;
2709 if (cfg
->dir
!= DMA_DEV_TO_MEM
)
2710 dev_dbg(d40c
->base
->dev
,
2711 "channel was not configured for peripheral "
2712 "to memory transfer (%d) overriding\n",
2714 cfg
->dir
= DMA_DEV_TO_MEM
;
2716 /* Configure the memory side */
2717 if (dst_addr_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
)
2718 dst_addr_width
= src_addr_width
;
2719 if (dst_maxburst
== 0)
2720 dst_maxburst
= src_maxburst
;
2722 } else if (config
->direction
== DMA_MEM_TO_DEV
) {
2723 config_addr
= config
->dst_addr
;
2725 if (cfg
->dir
!= DMA_MEM_TO_DEV
)
2726 dev_dbg(d40c
->base
->dev
,
2727 "channel was not configured for memory "
2728 "to peripheral transfer (%d) overriding\n",
2730 cfg
->dir
= DMA_MEM_TO_DEV
;
2732 /* Configure the memory side */
2733 if (src_addr_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
)
2734 src_addr_width
= dst_addr_width
;
2735 if (src_maxburst
== 0)
2736 src_maxburst
= dst_maxburst
;
2738 dev_err(d40c
->base
->dev
,
2739 "unrecognized channel direction %d\n",
2744 if (config_addr
<= 0) {
2745 dev_err(d40c
->base
->dev
, "no address supplied\n");
2749 if (src_maxburst
* src_addr_width
!= dst_maxburst
* dst_addr_width
) {
2750 dev_err(d40c
->base
->dev
,
2751 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2759 if (src_maxburst
> 16) {
2761 dst_maxburst
= src_maxburst
* src_addr_width
/ dst_addr_width
;
2762 } else if (dst_maxburst
> 16) {
2764 src_maxburst
= dst_maxburst
* dst_addr_width
/ src_addr_width
;
2767 /* Only valid widths are; 1, 2, 4 and 8. */
2768 if (src_addr_width
<= DMA_SLAVE_BUSWIDTH_UNDEFINED
||
2769 src_addr_width
> DMA_SLAVE_BUSWIDTH_8_BYTES
||
2770 dst_addr_width
<= DMA_SLAVE_BUSWIDTH_UNDEFINED
||
2771 dst_addr_width
> DMA_SLAVE_BUSWIDTH_8_BYTES
||
2772 !is_power_of_2(src_addr_width
) ||
2773 !is_power_of_2(dst_addr_width
))
2776 cfg
->src_info
.data_width
= src_addr_width
;
2777 cfg
->dst_info
.data_width
= dst_addr_width
;
2779 ret
= dma40_config_to_halfchannel(d40c
, &cfg
->src_info
,
2784 ret
= dma40_config_to_halfchannel(d40c
, &cfg
->dst_info
,
2789 /* Fill in register values */
2790 if (chan_is_logical(d40c
))
2791 d40_log_cfg(cfg
, &d40c
->log_def
.lcsp1
, &d40c
->log_def
.lcsp3
);
2793 d40_phy_cfg(cfg
, &d40c
->src_def_cfg
, &d40c
->dst_def_cfg
);
2795 /* These settings will take precedence later */
2796 d40c
->runtime_addr
= config_addr
;
2797 d40c
->runtime_direction
= config
->direction
;
2798 dev_dbg(d40c
->base
->dev
,
2799 "configured channel %s for %s, data width %d/%d, "
2800 "maxburst %d/%d elements, LE, no flow control\n",
2801 dma_chan_name(chan
),
2802 (config
->direction
== DMA_DEV_TO_MEM
) ? "RX" : "TX",
2803 src_addr_width
, dst_addr_width
,
2804 src_maxburst
, dst_maxburst
);
2809 /* Initialization functions */
2811 static void __init
d40_chan_init(struct d40_base
*base
, struct dma_device
*dma
,
2812 struct d40_chan
*chans
, int offset
,
2816 struct d40_chan
*d40c
;
2818 INIT_LIST_HEAD(&dma
->channels
);
2820 for (i
= offset
; i
< offset
+ num_chans
; i
++) {
2823 d40c
->chan
.device
= dma
;
2825 spin_lock_init(&d40c
->lock
);
2827 d40c
->log_num
= D40_PHY_CHAN
;
2829 INIT_LIST_HEAD(&d40c
->done
);
2830 INIT_LIST_HEAD(&d40c
->active
);
2831 INIT_LIST_HEAD(&d40c
->queue
);
2832 INIT_LIST_HEAD(&d40c
->pending_queue
);
2833 INIT_LIST_HEAD(&d40c
->client
);
2834 INIT_LIST_HEAD(&d40c
->prepare_queue
);
2836 tasklet_init(&d40c
->tasklet
, dma_tasklet
,
2837 (unsigned long) d40c
);
2839 list_add_tail(&d40c
->chan
.device_node
,
2844 static void d40_ops_init(struct d40_base
*base
, struct dma_device
*dev
)
2846 if (dma_has_cap(DMA_SLAVE
, dev
->cap_mask
))
2847 dev
->device_prep_slave_sg
= d40_prep_slave_sg
;
2849 if (dma_has_cap(DMA_MEMCPY
, dev
->cap_mask
)) {
2850 dev
->device_prep_dma_memcpy
= d40_prep_memcpy
;
2853 * This controller can only access address at even
2854 * 32bit boundaries, i.e. 2^2
2856 dev
->copy_align
= DMAENGINE_ALIGN_4_BYTES
;
2859 if (dma_has_cap(DMA_SG
, dev
->cap_mask
))
2860 dev
->device_prep_dma_sg
= d40_prep_memcpy_sg
;
2862 if (dma_has_cap(DMA_CYCLIC
, dev
->cap_mask
))
2863 dev
->device_prep_dma_cyclic
= dma40_prep_dma_cyclic
;
2865 dev
->device_alloc_chan_resources
= d40_alloc_chan_resources
;
2866 dev
->device_free_chan_resources
= d40_free_chan_resources
;
2867 dev
->device_issue_pending
= d40_issue_pending
;
2868 dev
->device_tx_status
= d40_tx_status
;
2869 dev
->device_config
= d40_set_runtime_config
;
2870 dev
->device_pause
= d40_pause
;
2871 dev
->device_resume
= d40_resume
;
2872 dev
->device_terminate_all
= d40_terminate_all
;
2873 dev
->dev
= base
->dev
;
2876 static int __init
d40_dmaengine_init(struct d40_base
*base
,
2877 int num_reserved_chans
)
2881 d40_chan_init(base
, &base
->dma_slave
, base
->log_chans
,
2882 0, base
->num_log_chans
);
2884 dma_cap_zero(base
->dma_slave
.cap_mask
);
2885 dma_cap_set(DMA_SLAVE
, base
->dma_slave
.cap_mask
);
2886 dma_cap_set(DMA_CYCLIC
, base
->dma_slave
.cap_mask
);
2888 d40_ops_init(base
, &base
->dma_slave
);
2890 err
= dma_async_device_register(&base
->dma_slave
);
2893 d40_err(base
->dev
, "Failed to register slave channels\n");
2897 d40_chan_init(base
, &base
->dma_memcpy
, base
->log_chans
,
2898 base
->num_log_chans
, base
->num_memcpy_chans
);
2900 dma_cap_zero(base
->dma_memcpy
.cap_mask
);
2901 dma_cap_set(DMA_MEMCPY
, base
->dma_memcpy
.cap_mask
);
2902 dma_cap_set(DMA_SG
, base
->dma_memcpy
.cap_mask
);
2904 d40_ops_init(base
, &base
->dma_memcpy
);
2906 err
= dma_async_device_register(&base
->dma_memcpy
);
2910 "Failed to register memcpy only channels\n");
2914 d40_chan_init(base
, &base
->dma_both
, base
->phy_chans
,
2915 0, num_reserved_chans
);
2917 dma_cap_zero(base
->dma_both
.cap_mask
);
2918 dma_cap_set(DMA_SLAVE
, base
->dma_both
.cap_mask
);
2919 dma_cap_set(DMA_MEMCPY
, base
->dma_both
.cap_mask
);
2920 dma_cap_set(DMA_SG
, base
->dma_both
.cap_mask
);
2921 dma_cap_set(DMA_CYCLIC
, base
->dma_slave
.cap_mask
);
2923 d40_ops_init(base
, &base
->dma_both
);
2924 err
= dma_async_device_register(&base
->dma_both
);
2928 "Failed to register logical and physical capable channels\n");
2933 dma_async_device_unregister(&base
->dma_memcpy
);
2935 dma_async_device_unregister(&base
->dma_slave
);
2940 /* Suspend resume functionality */
2941 #ifdef CONFIG_PM_SLEEP
2942 static int dma40_suspend(struct device
*dev
)
2944 struct platform_device
*pdev
= to_platform_device(dev
);
2945 struct d40_base
*base
= platform_get_drvdata(pdev
);
2948 ret
= pm_runtime_force_suspend(dev
);
2952 if (base
->lcpa_regulator
)
2953 ret
= regulator_disable(base
->lcpa_regulator
);
2957 static int dma40_resume(struct device
*dev
)
2959 struct platform_device
*pdev
= to_platform_device(dev
);
2960 struct d40_base
*base
= platform_get_drvdata(pdev
);
2963 if (base
->lcpa_regulator
) {
2964 ret
= regulator_enable(base
->lcpa_regulator
);
2969 return pm_runtime_force_resume(dev
);
2974 static void dma40_backup(void __iomem
*baseaddr
, u32
*backup
,
2975 u32
*regaddr
, int num
, bool save
)
2979 for (i
= 0; i
< num
; i
++) {
2980 void __iomem
*addr
= baseaddr
+ regaddr
[i
];
2983 backup
[i
] = readl_relaxed(addr
);
2985 writel_relaxed(backup
[i
], addr
);
2989 static void d40_save_restore_registers(struct d40_base
*base
, bool save
)
2993 /* Save/Restore channel specific registers */
2994 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
2998 if (base
->phy_res
[i
].reserved
)
3001 addr
= base
->virtbase
+ D40_DREG_PCBASE
+ i
* D40_DREG_PCDELTA
;
3002 idx
= i
* ARRAY_SIZE(d40_backup_regs_chan
);
3004 dma40_backup(addr
, &base
->reg_val_backup_chan
[idx
],
3005 d40_backup_regs_chan
,
3006 ARRAY_SIZE(d40_backup_regs_chan
),
3010 /* Save/Restore global registers */
3011 dma40_backup(base
->virtbase
, base
->reg_val_backup
,
3012 d40_backup_regs
, ARRAY_SIZE(d40_backup_regs
),
3015 /* Save/Restore registers only existing on dma40 v3 and later */
3016 if (base
->gen_dmac
.backup
)
3017 dma40_backup(base
->virtbase
, base
->reg_val_backup_v4
,
3018 base
->gen_dmac
.backup
,
3019 base
->gen_dmac
.backup_size
,
3023 static int dma40_runtime_suspend(struct device
*dev
)
3025 struct platform_device
*pdev
= to_platform_device(dev
);
3026 struct d40_base
*base
= platform_get_drvdata(pdev
);
3028 d40_save_restore_registers(base
, true);
3030 /* Don't disable/enable clocks for v1 due to HW bugs */
3032 writel_relaxed(base
->gcc_pwr_off_mask
,
3033 base
->virtbase
+ D40_DREG_GCC
);
3038 static int dma40_runtime_resume(struct device
*dev
)
3040 struct platform_device
*pdev
= to_platform_device(dev
);
3041 struct d40_base
*base
= platform_get_drvdata(pdev
);
3043 d40_save_restore_registers(base
, false);
3045 writel_relaxed(D40_DREG_GCC_ENABLE_ALL
,
3046 base
->virtbase
+ D40_DREG_GCC
);
3051 static const struct dev_pm_ops dma40_pm_ops
= {
3052 SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend
, dma40_resume
)
3053 SET_RUNTIME_PM_OPS(dma40_runtime_suspend
,
3054 dma40_runtime_resume
,
3058 /* Initialization functions. */
3060 static int __init
d40_phy_res_init(struct d40_base
*base
)
3063 int num_phy_chans_avail
= 0;
3065 int odd_even_bit
= -2;
3066 int gcc
= D40_DREG_GCC_ENA
;
3068 val
[0] = readl(base
->virtbase
+ D40_DREG_PRSME
);
3069 val
[1] = readl(base
->virtbase
+ D40_DREG_PRSMO
);
3071 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3072 base
->phy_res
[i
].num
= i
;
3073 odd_even_bit
+= 2 * ((i
% 2) == 0);
3074 if (((val
[i
% 2] >> odd_even_bit
) & 3) == 1) {
3075 /* Mark security only channels as occupied */
3076 base
->phy_res
[i
].allocated_src
= D40_ALLOC_PHY
;
3077 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_PHY
;
3078 base
->phy_res
[i
].reserved
= true;
3079 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i
),
3081 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i
),
3086 base
->phy_res
[i
].allocated_src
= D40_ALLOC_FREE
;
3087 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_FREE
;
3088 base
->phy_res
[i
].reserved
= false;
3089 num_phy_chans_avail
++;
3091 spin_lock_init(&base
->phy_res
[i
].lock
);
3094 /* Mark disabled channels as occupied */
3095 for (i
= 0; base
->plat_data
->disabled_channels
[i
] != -1; i
++) {
3096 int chan
= base
->plat_data
->disabled_channels
[i
];
3098 base
->phy_res
[chan
].allocated_src
= D40_ALLOC_PHY
;
3099 base
->phy_res
[chan
].allocated_dst
= D40_ALLOC_PHY
;
3100 base
->phy_res
[chan
].reserved
= true;
3101 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan
),
3103 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan
),
3105 num_phy_chans_avail
--;
3108 /* Mark soft_lli channels */
3109 for (i
= 0; i
< base
->plat_data
->num_of_soft_lli_chans
; i
++) {
3110 int chan
= base
->plat_data
->soft_lli_chans
[i
];
3112 base
->phy_res
[chan
].use_soft_lli
= true;
3115 dev_info(base
->dev
, "%d of %d physical DMA channels available\n",
3116 num_phy_chans_avail
, base
->num_phy_chans
);
3118 /* Verify settings extended vs standard */
3119 val
[0] = readl(base
->virtbase
+ D40_DREG_PRTYP
);
3121 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3123 if (base
->phy_res
[i
].allocated_src
== D40_ALLOC_FREE
&&
3124 (val
[0] & 0x3) != 1)
3126 "[%s] INFO: channel %d is misconfigured (%d)\n",
3127 __func__
, i
, val
[0] & 0x3);
3129 val
[0] = val
[0] >> 2;
3133 * To keep things simple, Enable all clocks initially.
3134 * The clocks will get managed later post channel allocation.
3135 * The clocks for the event lines on which reserved channels exists
3136 * are not managed here.
3138 writel(D40_DREG_GCC_ENABLE_ALL
, base
->virtbase
+ D40_DREG_GCC
);
3139 base
->gcc_pwr_off_mask
= gcc
;
3141 return num_phy_chans_avail
;
3144 static struct d40_base
* __init
d40_hw_detect_init(struct platform_device
*pdev
)
3146 struct stedma40_platform_data
*plat_data
= dev_get_platdata(&pdev
->dev
);
3148 void __iomem
*virtbase
;
3149 struct resource
*res
;
3150 struct d40_base
*base
;
3153 int num_memcpy_chans
;
3154 int clk_ret
= -EINVAL
;
3160 clk
= clk_get(&pdev
->dev
, NULL
);
3162 d40_err(&pdev
->dev
, "No matching clock found\n");
3163 goto check_prepare_enabled
;
3166 clk_ret
= clk_prepare_enable(clk
);
3168 d40_err(&pdev
->dev
, "Failed to prepare/enable clock\n");
3169 goto disable_unprepare
;
3172 /* Get IO for DMAC base address */
3173 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "base");
3175 goto disable_unprepare
;
3177 if (request_mem_region(res
->start
, resource_size(res
),
3178 D40_NAME
" I/O base") == NULL
)
3179 goto release_region
;
3181 virtbase
= ioremap(res
->start
, resource_size(res
));
3183 goto release_region
;
3185 /* This is just a regular AMBA PrimeCell ID actually */
3186 for (pid
= 0, i
= 0; i
< 4; i
++)
3187 pid
|= (readl(virtbase
+ resource_size(res
) - 0x20 + 4 * i
)
3189 for (cid
= 0, i
= 0; i
< 4; i
++)
3190 cid
|= (readl(virtbase
+ resource_size(res
) - 0x10 + 4 * i
)
3193 if (cid
!= AMBA_CID
) {
3194 d40_err(&pdev
->dev
, "Unknown hardware! No PrimeCell ID\n");
3197 if (AMBA_MANF_BITS(pid
) != AMBA_VENDOR_ST
) {
3198 d40_err(&pdev
->dev
, "Unknown designer! Got %x wanted %x\n",
3199 AMBA_MANF_BITS(pid
),
3205 * DB8500ed has revision 0
3207 * DB8500v1 has revision 2
3208 * DB8500v2 has revision 3
3209 * AP9540v1 has revision 4
3210 * DB8540v1 has revision 4
3212 rev
= AMBA_REV_BITS(pid
);
3214 d40_err(&pdev
->dev
, "hardware revision: %d is not supported", rev
);
3218 /* The number of physical channels on this HW */
3219 if (plat_data
->num_of_phy_chans
)
3220 num_phy_chans
= plat_data
->num_of_phy_chans
;
3222 num_phy_chans
= 4 * (readl(virtbase
+ D40_DREG_ICFG
) & 0x7) + 4;
3224 /* The number of channels used for memcpy */
3225 if (plat_data
->num_of_memcpy_chans
)
3226 num_memcpy_chans
= plat_data
->num_of_memcpy_chans
;
3228 num_memcpy_chans
= ARRAY_SIZE(dma40_memcpy_channels
);
3230 num_log_chans
= num_phy_chans
* D40_MAX_LOG_CHAN_PER_PHY
;
3232 dev_info(&pdev
->dev
,
3233 "hardware rev: %d @ %pa with %d physical and %d logical channels\n",
3234 rev
, &res
->start
, num_phy_chans
, num_log_chans
);
3236 base
= kzalloc(ALIGN(sizeof(struct d40_base
), 4) +
3237 (num_phy_chans
+ num_log_chans
+ num_memcpy_chans
) *
3238 sizeof(struct d40_chan
), GFP_KERNEL
);
3245 base
->num_memcpy_chans
= num_memcpy_chans
;
3246 base
->num_phy_chans
= num_phy_chans
;
3247 base
->num_log_chans
= num_log_chans
;
3248 base
->phy_start
= res
->start
;
3249 base
->phy_size
= resource_size(res
);
3250 base
->virtbase
= virtbase
;
3251 base
->plat_data
= plat_data
;
3252 base
->dev
= &pdev
->dev
;
3253 base
->phy_chans
= ((void *)base
) + ALIGN(sizeof(struct d40_base
), 4);
3254 base
->log_chans
= &base
->phy_chans
[num_phy_chans
];
3256 if (base
->plat_data
->num_of_phy_chans
== 14) {
3257 base
->gen_dmac
.backup
= d40_backup_regs_v4b
;
3258 base
->gen_dmac
.backup_size
= BACKUP_REGS_SZ_V4B
;
3259 base
->gen_dmac
.interrupt_en
= D40_DREG_CPCMIS
;
3260 base
->gen_dmac
.interrupt_clear
= D40_DREG_CPCICR
;
3261 base
->gen_dmac
.realtime_en
= D40_DREG_CRSEG1
;
3262 base
->gen_dmac
.realtime_clear
= D40_DREG_CRCEG1
;
3263 base
->gen_dmac
.high_prio_en
= D40_DREG_CPSEG1
;
3264 base
->gen_dmac
.high_prio_clear
= D40_DREG_CPCEG1
;
3265 base
->gen_dmac
.il
= il_v4b
;
3266 base
->gen_dmac
.il_size
= ARRAY_SIZE(il_v4b
);
3267 base
->gen_dmac
.init_reg
= dma_init_reg_v4b
;
3268 base
->gen_dmac
.init_reg_size
= ARRAY_SIZE(dma_init_reg_v4b
);
3270 if (base
->rev
>= 3) {
3271 base
->gen_dmac
.backup
= d40_backup_regs_v4a
;
3272 base
->gen_dmac
.backup_size
= BACKUP_REGS_SZ_V4A
;
3274 base
->gen_dmac
.interrupt_en
= D40_DREG_PCMIS
;
3275 base
->gen_dmac
.interrupt_clear
= D40_DREG_PCICR
;
3276 base
->gen_dmac
.realtime_en
= D40_DREG_RSEG1
;
3277 base
->gen_dmac
.realtime_clear
= D40_DREG_RCEG1
;
3278 base
->gen_dmac
.high_prio_en
= D40_DREG_PSEG1
;
3279 base
->gen_dmac
.high_prio_clear
= D40_DREG_PCEG1
;
3280 base
->gen_dmac
.il
= il_v4a
;
3281 base
->gen_dmac
.il_size
= ARRAY_SIZE(il_v4a
);
3282 base
->gen_dmac
.init_reg
= dma_init_reg_v4a
;
3283 base
->gen_dmac
.init_reg_size
= ARRAY_SIZE(dma_init_reg_v4a
);
3286 base
->phy_res
= kcalloc(num_phy_chans
,
3287 sizeof(*base
->phy_res
),
3292 base
->lookup_phy_chans
= kcalloc(num_phy_chans
,
3293 sizeof(*base
->lookup_phy_chans
),
3295 if (!base
->lookup_phy_chans
)
3298 base
->lookup_log_chans
= kcalloc(num_log_chans
,
3299 sizeof(*base
->lookup_log_chans
),
3301 if (!base
->lookup_log_chans
)
3302 goto free_phy_chans
;
3304 base
->reg_val_backup_chan
= kmalloc_array(base
->num_phy_chans
,
3305 sizeof(d40_backup_regs_chan
),
3307 if (!base
->reg_val_backup_chan
)
3308 goto free_log_chans
;
3310 base
->lcla_pool
.alloc_map
= kcalloc(num_phy_chans
3311 * D40_LCLA_LINK_PER_EVENT_GRP
,
3312 sizeof(*base
->lcla_pool
.alloc_map
),
3314 if (!base
->lcla_pool
.alloc_map
)
3315 goto free_backup_chan
;
3317 base
->desc_slab
= kmem_cache_create(D40_NAME
, sizeof(struct d40_desc
),
3318 0, SLAB_HWCACHE_ALIGN
,
3320 if (base
->desc_slab
== NULL
)
3325 kfree(base
->lcla_pool
.alloc_map
);
3327 kfree(base
->reg_val_backup_chan
);
3329 kfree(base
->lookup_log_chans
);
3331 kfree(base
->lookup_phy_chans
);
3333 kfree(base
->phy_res
);
3339 release_mem_region(res
->start
, resource_size(res
));
3340 check_prepare_enabled
:
3343 clk_disable_unprepare(clk
);
3349 static void __init
d40_hw_init(struct d40_base
*base
)
3353 u32 prmseo
[2] = {0, 0};
3354 u32 activeo
[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3357 struct d40_reg_val
*dma_init_reg
= base
->gen_dmac
.init_reg
;
3358 u32 reg_size
= base
->gen_dmac
.init_reg_size
;
3360 for (i
= 0; i
< reg_size
; i
++)
3361 writel(dma_init_reg
[i
].val
,
3362 base
->virtbase
+ dma_init_reg
[i
].reg
);
3364 /* Configure all our dma channels to default settings */
3365 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3367 activeo
[i
% 2] = activeo
[i
% 2] << 2;
3369 if (base
->phy_res
[base
->num_phy_chans
- i
- 1].allocated_src
3371 activeo
[i
% 2] |= 3;
3375 /* Enable interrupt # */
3376 pcmis
= (pcmis
<< 1) | 1;
3378 /* Clear interrupt # */
3379 pcicr
= (pcicr
<< 1) | 1;
3381 /* Set channel to physical mode */
3382 prmseo
[i
% 2] = prmseo
[i
% 2] << 2;
3387 writel(prmseo
[1], base
->virtbase
+ D40_DREG_PRMSE
);
3388 writel(prmseo
[0], base
->virtbase
+ D40_DREG_PRMSO
);
3389 writel(activeo
[1], base
->virtbase
+ D40_DREG_ACTIVE
);
3390 writel(activeo
[0], base
->virtbase
+ D40_DREG_ACTIVO
);
3392 /* Write which interrupt to enable */
3393 writel(pcmis
, base
->virtbase
+ base
->gen_dmac
.interrupt_en
);
3395 /* Write which interrupt to clear */
3396 writel(pcicr
, base
->virtbase
+ base
->gen_dmac
.interrupt_clear
);
3398 /* These are __initdata and cannot be accessed after init */
3399 base
->gen_dmac
.init_reg
= NULL
;
3400 base
->gen_dmac
.init_reg_size
= 0;
3403 static int __init
d40_lcla_allocate(struct d40_base
*base
)
3405 struct d40_lcla_pool
*pool
= &base
->lcla_pool
;
3406 unsigned long *page_list
;
3411 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3412 * To full fill this hardware requirement without wasting 256 kb
3413 * we allocate pages until we get an aligned one.
3415 page_list
= kmalloc_array(MAX_LCLA_ALLOC_ATTEMPTS
,
3421 /* Calculating how many pages that are required */
3422 base
->lcla_pool
.pages
= SZ_1K
* base
->num_phy_chans
/ PAGE_SIZE
;
3424 for (i
= 0; i
< MAX_LCLA_ALLOC_ATTEMPTS
; i
++) {
3425 page_list
[i
] = __get_free_pages(GFP_KERNEL
,
3426 base
->lcla_pool
.pages
);
3427 if (!page_list
[i
]) {
3429 d40_err(base
->dev
, "Failed to allocate %d pages.\n",
3430 base
->lcla_pool
.pages
);
3433 for (j
= 0; j
< i
; j
++)
3434 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
3435 goto free_page_list
;
3438 if ((virt_to_phys((void *)page_list
[i
]) &
3439 (LCLA_ALIGNMENT
- 1)) == 0)
3443 for (j
= 0; j
< i
; j
++)
3444 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
3446 if (i
< MAX_LCLA_ALLOC_ATTEMPTS
) {
3447 base
->lcla_pool
.base
= (void *)page_list
[i
];
3450 * After many attempts and no succees with finding the correct
3451 * alignment, try with allocating a big buffer.
3454 "[%s] Failed to get %d pages @ 18 bit align.\n",
3455 __func__
, base
->lcla_pool
.pages
);
3456 base
->lcla_pool
.base_unaligned
= kmalloc(SZ_1K
*
3457 base
->num_phy_chans
+
3460 if (!base
->lcla_pool
.base_unaligned
) {
3462 goto free_page_list
;
3465 base
->lcla_pool
.base
= PTR_ALIGN(base
->lcla_pool
.base_unaligned
,
3469 pool
->dma_addr
= dma_map_single(base
->dev
, pool
->base
,
3470 SZ_1K
* base
->num_phy_chans
,
3472 if (dma_mapping_error(base
->dev
, pool
->dma_addr
)) {
3475 goto free_page_list
;
3478 writel(virt_to_phys(base
->lcla_pool
.base
),
3479 base
->virtbase
+ D40_DREG_LCLA
);
3486 static int __init
d40_of_probe(struct platform_device
*pdev
,
3487 struct device_node
*np
)
3489 struct stedma40_platform_data
*pdata
;
3490 int num_phy
= 0, num_memcpy
= 0, num_disabled
= 0;
3493 pdata
= devm_kzalloc(&pdev
->dev
, sizeof(*pdata
), GFP_KERNEL
);
3497 /* If absent this value will be obtained from h/w. */
3498 of_property_read_u32(np
, "dma-channels", &num_phy
);
3500 pdata
->num_of_phy_chans
= num_phy
;
3502 list
= of_get_property(np
, "memcpy-channels", &num_memcpy
);
3503 num_memcpy
/= sizeof(*list
);
3505 if (num_memcpy
> D40_MEMCPY_MAX_CHANS
|| num_memcpy
<= 0) {
3507 "Invalid number of memcpy channels specified (%d)\n",
3511 pdata
->num_of_memcpy_chans
= num_memcpy
;
3513 of_property_read_u32_array(np
, "memcpy-channels",
3514 dma40_memcpy_channels
,
3517 list
= of_get_property(np
, "disabled-channels", &num_disabled
);
3518 num_disabled
/= sizeof(*list
);
3520 if (num_disabled
>= STEDMA40_MAX_PHYS
|| num_disabled
< 0) {
3522 "Invalid number of disabled channels specified (%d)\n",
3527 of_property_read_u32_array(np
, "disabled-channels",
3528 pdata
->disabled_channels
,
3530 pdata
->disabled_channels
[num_disabled
] = -1;
3532 pdev
->dev
.platform_data
= pdata
;
3537 static int __init
d40_probe(struct platform_device
*pdev
)
3539 struct stedma40_platform_data
*plat_data
= dev_get_platdata(&pdev
->dev
);
3540 struct device_node
*np
= pdev
->dev
.of_node
;
3542 struct d40_base
*base
;
3543 struct resource
*res
;
3544 int num_reserved_chans
;
3549 if (d40_of_probe(pdev
, np
)) {
3551 goto report_failure
;
3554 d40_err(&pdev
->dev
, "No pdata or Device Tree provided\n");
3555 goto report_failure
;
3559 base
= d40_hw_detect_init(pdev
);
3561 goto report_failure
;
3563 num_reserved_chans
= d40_phy_res_init(base
);
3565 platform_set_drvdata(pdev
, base
);
3567 spin_lock_init(&base
->interrupt_lock
);
3568 spin_lock_init(&base
->execmd_lock
);
3570 /* Get IO for logical channel parameter address */
3571 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "lcpa");
3574 d40_err(&pdev
->dev
, "No \"lcpa\" memory resource\n");
3577 base
->lcpa_size
= resource_size(res
);
3578 base
->phy_lcpa
= res
->start
;
3580 if (request_mem_region(res
->start
, resource_size(res
),
3581 D40_NAME
" I/O lcpa") == NULL
) {
3583 d40_err(&pdev
->dev
, "Failed to request LCPA region %pR\n", res
);
3587 /* We make use of ESRAM memory for this. */
3588 val
= readl(base
->virtbase
+ D40_DREG_LCPA
);
3589 if (res
->start
!= val
&& val
!= 0) {
3590 dev_warn(&pdev
->dev
,
3591 "[%s] Mismatch LCPA dma 0x%x, def %pa\n",
3592 __func__
, val
, &res
->start
);
3594 writel(res
->start
, base
->virtbase
+ D40_DREG_LCPA
);
3596 base
->lcpa_base
= ioremap(res
->start
, resource_size(res
));
3597 if (!base
->lcpa_base
) {
3599 d40_err(&pdev
->dev
, "Failed to ioremap LCPA region\n");
3602 /* If lcla has to be located in ESRAM we don't need to allocate */
3603 if (base
->plat_data
->use_esram_lcla
) {
3604 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
,
3609 "No \"lcla_esram\" memory resource\n");
3612 base
->lcla_pool
.base
= ioremap(res
->start
,
3613 resource_size(res
));
3614 if (!base
->lcla_pool
.base
) {
3616 d40_err(&pdev
->dev
, "Failed to ioremap LCLA region\n");
3619 writel(res
->start
, base
->virtbase
+ D40_DREG_LCLA
);
3622 ret
= d40_lcla_allocate(base
);
3624 d40_err(&pdev
->dev
, "Failed to allocate LCLA area\n");
3629 spin_lock_init(&base
->lcla_pool
.lock
);
3631 base
->irq
= platform_get_irq(pdev
, 0);
3633 ret
= request_irq(base
->irq
, d40_handle_interrupt
, 0, D40_NAME
, base
);
3635 d40_err(&pdev
->dev
, "No IRQ defined\n");
3639 if (base
->plat_data
->use_esram_lcla
) {
3641 base
->lcpa_regulator
= regulator_get(base
->dev
, "lcla_esram");
3642 if (IS_ERR(base
->lcpa_regulator
)) {
3643 d40_err(&pdev
->dev
, "Failed to get lcpa_regulator\n");
3644 ret
= PTR_ERR(base
->lcpa_regulator
);
3645 base
->lcpa_regulator
= NULL
;
3649 ret
= regulator_enable(base
->lcpa_regulator
);
3652 "Failed to enable lcpa_regulator\n");
3653 regulator_put(base
->lcpa_regulator
);
3654 base
->lcpa_regulator
= NULL
;
3659 writel_relaxed(D40_DREG_GCC_ENABLE_ALL
, base
->virtbase
+ D40_DREG_GCC
);
3661 pm_runtime_irq_safe(base
->dev
);
3662 pm_runtime_set_autosuspend_delay(base
->dev
, DMA40_AUTOSUSPEND_DELAY
);
3663 pm_runtime_use_autosuspend(base
->dev
);
3664 pm_runtime_mark_last_busy(base
->dev
);
3665 pm_runtime_set_active(base
->dev
);
3666 pm_runtime_enable(base
->dev
);
3668 ret
= d40_dmaengine_init(base
, num_reserved_chans
);
3672 base
->dev
->dma_parms
= &base
->dma_parms
;
3673 ret
= dma_set_max_seg_size(base
->dev
, STEDMA40_MAX_SEG_SIZE
);
3675 d40_err(&pdev
->dev
, "Failed to set dma max seg size\n");
3682 ret
= of_dma_controller_register(np
, d40_xlate
, NULL
);
3685 "could not register of_dma_controller\n");
3688 dev_info(base
->dev
, "initialized\n");
3691 kmem_cache_destroy(base
->desc_slab
);
3693 iounmap(base
->virtbase
);
3695 if (base
->lcla_pool
.base
&& base
->plat_data
->use_esram_lcla
) {
3696 iounmap(base
->lcla_pool
.base
);
3697 base
->lcla_pool
.base
= NULL
;
3700 if (base
->lcla_pool
.dma_addr
)
3701 dma_unmap_single(base
->dev
, base
->lcla_pool
.dma_addr
,
3702 SZ_1K
* base
->num_phy_chans
,
3705 if (!base
->lcla_pool
.base_unaligned
&& base
->lcla_pool
.base
)
3706 free_pages((unsigned long)base
->lcla_pool
.base
,
3707 base
->lcla_pool
.pages
);
3709 kfree(base
->lcla_pool
.base_unaligned
);
3712 release_mem_region(base
->phy_lcpa
,
3714 if (base
->phy_start
)
3715 release_mem_region(base
->phy_start
,
3718 clk_disable_unprepare(base
->clk
);
3722 if (base
->lcpa_regulator
) {
3723 regulator_disable(base
->lcpa_regulator
);
3724 regulator_put(base
->lcpa_regulator
);
3727 kfree(base
->lcla_pool
.alloc_map
);
3728 kfree(base
->lookup_log_chans
);
3729 kfree(base
->lookup_phy_chans
);
3730 kfree(base
->phy_res
);
3733 d40_err(&pdev
->dev
, "probe failed\n");
3737 static const struct of_device_id d40_match
[] = {
3738 { .compatible
= "stericsson,dma40", },
3742 static struct platform_driver d40_driver
= {
3745 .pm
= &dma40_pm_ops
,
3746 .of_match_table
= d40_match
,
3750 static int __init
stedma40_init(void)
3752 return platform_driver_probe(&d40_driver
, d40_probe
);
3754 subsys_initcall(stedma40_init
);