2 * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 and
6 * only version 2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 * QCOM BAM DMA engine driver
17 * QCOM BAM DMA blocks are distributed amongst a number of the on-chip
18 * peripherals on the MSM 8x74. The configuration of the channels are dependent
19 * on the way they are hard wired to that specific peripheral. The peripheral
20 * device tree entries specify the configuration of each channel.
22 * The DMA controller requires the use of external memory for storage of the
23 * hardware descriptors for each channel. The descriptor FIFO is accessed as a
24 * circular buffer and operations are managed according to the offset within the
25 * FIFO. After pipe/channel reset, all of the pipe registers and internal state
26 * are back to defaults.
28 * During DMA operations, we write descriptors to the FIFO, being careful to
29 * handle wrapping and then write the last FIFO offset to that channel's
30 * P_EVNT_REG register to kick off the transaction. The P_SW_OFSTS register
31 * indicates the current FIFO offset that is being processed, so there is some
32 * indication of where the hardware is currently working.
35 #include <linux/kernel.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/module.h>
40 #include <linux/interrupt.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/scatterlist.h>
43 #include <linux/device.h>
44 #include <linux/platform_device.h>
46 #include <linux/of_address.h>
47 #include <linux/of_irq.h>
48 #include <linux/of_dma.h>
49 #include <linux/circ_buf.h>
50 #include <linux/clk.h>
51 #include <linux/dmaengine.h>
52 #include <linux/pm_runtime.h>
54 #include "../dmaengine.h"
55 #include "../virt-dma.h"
58 __le32 addr
; /* Buffer physical address */
59 __le16 size
; /* Buffer size in bytes */
63 #define BAM_DMA_AUTOSUSPEND_DELAY 100
65 #define DESC_FLAG_INT BIT(15)
66 #define DESC_FLAG_EOT BIT(14)
67 #define DESC_FLAG_EOB BIT(13)
68 #define DESC_FLAG_NWD BIT(12)
69 #define DESC_FLAG_CMD BIT(11)
71 struct bam_async_desc
{
72 struct virt_dma_desc vd
;
77 /* transaction flags, EOT|EOB|NWD */
80 struct bam_desc_hw
*curr_desc
;
82 /* list node for the desc in the bam_chan list of descriptors */
83 struct list_head desc_node
;
84 enum dma_transfer_direction dir
;
86 struct bam_desc_hw desc
[0];
96 BAM_IRQ_SRCS_UNMASKED
,
109 BAM_P_EVNT_DEST_ADDR
,
112 BAM_P_DATA_FIFO_ADDR
,
113 BAM_P_DESC_FIFO_ADDR
,
114 BAM_P_EVNT_GEN_TRSHLD
,
118 struct reg_offset_data
{
120 unsigned int pipe_mult
, evnt_mult
, ee_mult
;
123 static const struct reg_offset_data bam_v1_3_reg_info
[] = {
124 [BAM_CTRL
] = { 0x0F80, 0x00, 0x00, 0x00 },
125 [BAM_REVISION
] = { 0x0F84, 0x00, 0x00, 0x00 },
126 [BAM_NUM_PIPES
] = { 0x0FBC, 0x00, 0x00, 0x00 },
127 [BAM_DESC_CNT_TRSHLD
] = { 0x0F88, 0x00, 0x00, 0x00 },
128 [BAM_IRQ_SRCS
] = { 0x0F8C, 0x00, 0x00, 0x00 },
129 [BAM_IRQ_SRCS_MSK
] = { 0x0F90, 0x00, 0x00, 0x00 },
130 [BAM_IRQ_SRCS_UNMASKED
] = { 0x0FB0, 0x00, 0x00, 0x00 },
131 [BAM_IRQ_STTS
] = { 0x0F94, 0x00, 0x00, 0x00 },
132 [BAM_IRQ_CLR
] = { 0x0F98, 0x00, 0x00, 0x00 },
133 [BAM_IRQ_EN
] = { 0x0F9C, 0x00, 0x00, 0x00 },
134 [BAM_CNFG_BITS
] = { 0x0FFC, 0x00, 0x00, 0x00 },
135 [BAM_IRQ_SRCS_EE
] = { 0x1800, 0x00, 0x00, 0x80 },
136 [BAM_IRQ_SRCS_MSK_EE
] = { 0x1804, 0x00, 0x00, 0x80 },
137 [BAM_P_CTRL
] = { 0x0000, 0x80, 0x00, 0x00 },
138 [BAM_P_RST
] = { 0x0004, 0x80, 0x00, 0x00 },
139 [BAM_P_HALT
] = { 0x0008, 0x80, 0x00, 0x00 },
140 [BAM_P_IRQ_STTS
] = { 0x0010, 0x80, 0x00, 0x00 },
141 [BAM_P_IRQ_CLR
] = { 0x0014, 0x80, 0x00, 0x00 },
142 [BAM_P_IRQ_EN
] = { 0x0018, 0x80, 0x00, 0x00 },
143 [BAM_P_EVNT_DEST_ADDR
] = { 0x102C, 0x00, 0x40, 0x00 },
144 [BAM_P_EVNT_REG
] = { 0x1018, 0x00, 0x40, 0x00 },
145 [BAM_P_SW_OFSTS
] = { 0x1000, 0x00, 0x40, 0x00 },
146 [BAM_P_DATA_FIFO_ADDR
] = { 0x1024, 0x00, 0x40, 0x00 },
147 [BAM_P_DESC_FIFO_ADDR
] = { 0x101C, 0x00, 0x40, 0x00 },
148 [BAM_P_EVNT_GEN_TRSHLD
] = { 0x1028, 0x00, 0x40, 0x00 },
149 [BAM_P_FIFO_SIZES
] = { 0x1020, 0x00, 0x40, 0x00 },
152 static const struct reg_offset_data bam_v1_4_reg_info
[] = {
153 [BAM_CTRL
] = { 0x0000, 0x00, 0x00, 0x00 },
154 [BAM_REVISION
] = { 0x0004, 0x00, 0x00, 0x00 },
155 [BAM_NUM_PIPES
] = { 0x003C, 0x00, 0x00, 0x00 },
156 [BAM_DESC_CNT_TRSHLD
] = { 0x0008, 0x00, 0x00, 0x00 },
157 [BAM_IRQ_SRCS
] = { 0x000C, 0x00, 0x00, 0x00 },
158 [BAM_IRQ_SRCS_MSK
] = { 0x0010, 0x00, 0x00, 0x00 },
159 [BAM_IRQ_SRCS_UNMASKED
] = { 0x0030, 0x00, 0x00, 0x00 },
160 [BAM_IRQ_STTS
] = { 0x0014, 0x00, 0x00, 0x00 },
161 [BAM_IRQ_CLR
] = { 0x0018, 0x00, 0x00, 0x00 },
162 [BAM_IRQ_EN
] = { 0x001C, 0x00, 0x00, 0x00 },
163 [BAM_CNFG_BITS
] = { 0x007C, 0x00, 0x00, 0x00 },
164 [BAM_IRQ_SRCS_EE
] = { 0x0800, 0x00, 0x00, 0x80 },
165 [BAM_IRQ_SRCS_MSK_EE
] = { 0x0804, 0x00, 0x00, 0x80 },
166 [BAM_P_CTRL
] = { 0x1000, 0x1000, 0x00, 0x00 },
167 [BAM_P_RST
] = { 0x1004, 0x1000, 0x00, 0x00 },
168 [BAM_P_HALT
] = { 0x1008, 0x1000, 0x00, 0x00 },
169 [BAM_P_IRQ_STTS
] = { 0x1010, 0x1000, 0x00, 0x00 },
170 [BAM_P_IRQ_CLR
] = { 0x1014, 0x1000, 0x00, 0x00 },
171 [BAM_P_IRQ_EN
] = { 0x1018, 0x1000, 0x00, 0x00 },
172 [BAM_P_EVNT_DEST_ADDR
] = { 0x182C, 0x00, 0x1000, 0x00 },
173 [BAM_P_EVNT_REG
] = { 0x1818, 0x00, 0x1000, 0x00 },
174 [BAM_P_SW_OFSTS
] = { 0x1800, 0x00, 0x1000, 0x00 },
175 [BAM_P_DATA_FIFO_ADDR
] = { 0x1824, 0x00, 0x1000, 0x00 },
176 [BAM_P_DESC_FIFO_ADDR
] = { 0x181C, 0x00, 0x1000, 0x00 },
177 [BAM_P_EVNT_GEN_TRSHLD
] = { 0x1828, 0x00, 0x1000, 0x00 },
178 [BAM_P_FIFO_SIZES
] = { 0x1820, 0x00, 0x1000, 0x00 },
181 static const struct reg_offset_data bam_v1_7_reg_info
[] = {
182 [BAM_CTRL
] = { 0x00000, 0x00, 0x00, 0x00 },
183 [BAM_REVISION
] = { 0x01000, 0x00, 0x00, 0x00 },
184 [BAM_NUM_PIPES
] = { 0x01008, 0x00, 0x00, 0x00 },
185 [BAM_DESC_CNT_TRSHLD
] = { 0x00008, 0x00, 0x00, 0x00 },
186 [BAM_IRQ_SRCS
] = { 0x03010, 0x00, 0x00, 0x00 },
187 [BAM_IRQ_SRCS_MSK
] = { 0x03014, 0x00, 0x00, 0x00 },
188 [BAM_IRQ_SRCS_UNMASKED
] = { 0x03018, 0x00, 0x00, 0x00 },
189 [BAM_IRQ_STTS
] = { 0x00014, 0x00, 0x00, 0x00 },
190 [BAM_IRQ_CLR
] = { 0x00018, 0x00, 0x00, 0x00 },
191 [BAM_IRQ_EN
] = { 0x0001C, 0x00, 0x00, 0x00 },
192 [BAM_CNFG_BITS
] = { 0x0007C, 0x00, 0x00, 0x00 },
193 [BAM_IRQ_SRCS_EE
] = { 0x03000, 0x00, 0x00, 0x1000 },
194 [BAM_IRQ_SRCS_MSK_EE
] = { 0x03004, 0x00, 0x00, 0x1000 },
195 [BAM_P_CTRL
] = { 0x13000, 0x1000, 0x00, 0x00 },
196 [BAM_P_RST
] = { 0x13004, 0x1000, 0x00, 0x00 },
197 [BAM_P_HALT
] = { 0x13008, 0x1000, 0x00, 0x00 },
198 [BAM_P_IRQ_STTS
] = { 0x13010, 0x1000, 0x00, 0x00 },
199 [BAM_P_IRQ_CLR
] = { 0x13014, 0x1000, 0x00, 0x00 },
200 [BAM_P_IRQ_EN
] = { 0x13018, 0x1000, 0x00, 0x00 },
201 [BAM_P_EVNT_DEST_ADDR
] = { 0x1382C, 0x00, 0x1000, 0x00 },
202 [BAM_P_EVNT_REG
] = { 0x13818, 0x00, 0x1000, 0x00 },
203 [BAM_P_SW_OFSTS
] = { 0x13800, 0x00, 0x1000, 0x00 },
204 [BAM_P_DATA_FIFO_ADDR
] = { 0x13824, 0x00, 0x1000, 0x00 },
205 [BAM_P_DESC_FIFO_ADDR
] = { 0x1381C, 0x00, 0x1000, 0x00 },
206 [BAM_P_EVNT_GEN_TRSHLD
] = { 0x13828, 0x00, 0x1000, 0x00 },
207 [BAM_P_FIFO_SIZES
] = { 0x13820, 0x00, 0x1000, 0x00 },
211 #define BAM_SW_RST BIT(0)
212 #define BAM_EN BIT(1)
213 #define BAM_EN_ACCUM BIT(4)
214 #define BAM_TESTBUS_SEL_SHIFT 5
215 #define BAM_TESTBUS_SEL_MASK 0x3F
216 #define BAM_DESC_CACHE_SEL_SHIFT 13
217 #define BAM_DESC_CACHE_SEL_MASK 0x3
218 #define BAM_CACHED_DESC_STORE BIT(15)
219 #define IBC_DISABLE BIT(16)
222 #define REVISION_SHIFT 0
223 #define REVISION_MASK 0xFF
224 #define NUM_EES_SHIFT 8
225 #define NUM_EES_MASK 0xF
226 #define CE_BUFFER_SIZE BIT(13)
227 #define AXI_ACTIVE BIT(14)
228 #define USE_VMIDMT BIT(15)
229 #define SECURED BIT(16)
230 #define BAM_HAS_NO_BYPASS BIT(17)
231 #define HIGH_FREQUENCY_BAM BIT(18)
232 #define INACTIV_TMRS_EXST BIT(19)
233 #define NUM_INACTIV_TMRS BIT(20)
234 #define DESC_CACHE_DEPTH_SHIFT 21
235 #define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT)
236 #define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT)
237 #define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT)
238 #define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT)
239 #define CMD_DESC_EN BIT(23)
240 #define INACTIV_TMR_BASE_SHIFT 24
241 #define INACTIV_TMR_BASE_MASK 0xFF
244 #define BAM_NUM_PIPES_SHIFT 0
245 #define BAM_NUM_PIPES_MASK 0xFF
246 #define PERIPH_NON_PIPE_GRP_SHIFT 16
247 #define PERIPH_NON_PIP_GRP_MASK 0xFF
248 #define BAM_NON_PIPE_GRP_SHIFT 24
249 #define BAM_NON_PIPE_GRP_MASK 0xFF
252 #define BAM_PIPE_CNFG BIT(2)
253 #define BAM_FULL_PIPE BIT(11)
254 #define BAM_NO_EXT_P_RST BIT(12)
255 #define BAM_IBC_DISABLE BIT(13)
256 #define BAM_SB_CLK_REQ BIT(14)
257 #define BAM_PSM_CSW_REQ BIT(15)
258 #define BAM_PSM_P_RES BIT(16)
259 #define BAM_AU_P_RES BIT(17)
260 #define BAM_SI_P_RES BIT(18)
261 #define BAM_WB_P_RES BIT(19)
262 #define BAM_WB_BLK_CSW BIT(20)
263 #define BAM_WB_CSW_ACK_IDL BIT(21)
264 #define BAM_WB_RETR_SVPNT BIT(22)
265 #define BAM_WB_DSC_AVL_P_RST BIT(23)
266 #define BAM_REG_P_EN BIT(24)
267 #define BAM_PSM_P_HD_DATA BIT(25)
268 #define BAM_AU_ACCUMED BIT(26)
269 #define BAM_CMD_ENABLE BIT(27)
271 #define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \
281 BAM_WB_CSW_ACK_IDL | \
282 BAM_WB_RETR_SVPNT | \
283 BAM_WB_DSC_AVL_P_RST | \
285 BAM_PSM_P_HD_DATA | \
291 #define P_DIRECTION BIT(3)
292 #define P_SYS_STRM BIT(4)
293 #define P_SYS_MODE BIT(5)
294 #define P_AUTO_EOB BIT(6)
295 #define P_AUTO_EOB_SEL_SHIFT 7
296 #define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT)
297 #define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT)
298 #define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT)
299 #define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT)
300 #define P_PREFETCH_LIMIT_SHIFT 9
301 #define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT)
302 #define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT)
303 #define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT)
304 #define P_WRITE_NWD BIT(11)
305 #define P_LOCK_GROUP_SHIFT 16
306 #define P_LOCK_GROUP_MASK 0x1F
308 /* BAM_DESC_CNT_TRSHLD */
309 #define CNT_TRSHLD 0xffff
310 #define DEFAULT_CNT_THRSHLD 0x4
313 #define BAM_IRQ BIT(31)
314 #define P_IRQ 0x7fffffff
316 /* BAM_IRQ_SRCS_MSK */
317 #define BAM_IRQ_MSK BAM_IRQ
318 #define P_IRQ_MSK P_IRQ
321 #define BAM_TIMER_IRQ BIT(4)
322 #define BAM_EMPTY_IRQ BIT(3)
323 #define BAM_ERROR_IRQ BIT(2)
324 #define BAM_HRESP_ERR_IRQ BIT(1)
327 #define BAM_TIMER_CLR BIT(4)
328 #define BAM_EMPTY_CLR BIT(3)
329 #define BAM_ERROR_CLR BIT(2)
330 #define BAM_HRESP_ERR_CLR BIT(1)
333 #define BAM_TIMER_EN BIT(4)
334 #define BAM_EMPTY_EN BIT(3)
335 #define BAM_ERROR_EN BIT(2)
336 #define BAM_HRESP_ERR_EN BIT(1)
339 #define P_PRCSD_DESC_EN BIT(0)
340 #define P_TIMER_EN BIT(1)
341 #define P_WAKE_EN BIT(2)
342 #define P_OUT_OF_DESC_EN BIT(3)
343 #define P_ERR_EN BIT(4)
344 #define P_TRNSFR_END_EN BIT(5)
345 #define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
348 #define P_SW_OFSTS_MASK 0xffff
350 #define BAM_DESC_FIFO_SIZE SZ_32K
351 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
352 #define BAM_FIFO_SIZE (SZ_32K - 8)
353 #define IS_BUSY(chan) (CIRC_SPACE(bchan->tail, bchan->head,\
354 MAX_DESCRIPTORS + 1) == 0)
357 struct virt_dma_chan vc
;
359 struct bam_device
*bdev
;
361 /* configuration from device tree */
364 /* runtime configuration */
365 struct dma_slave_config slave
;
368 struct bam_desc_hw
*fifo_virt
;
369 dma_addr_t fifo_phys
;
372 unsigned short head
; /* start of active descriptor entries */
373 unsigned short tail
; /* end of active descriptor entries */
375 unsigned int initialized
; /* is the channel hw initialized? */
376 unsigned int paused
; /* is the channel paused? */
377 unsigned int reconfigure
; /* new slave config? */
378 /* list of descriptors currently processed */
379 struct list_head desc_list
;
381 struct list_head node
;
384 static inline struct bam_chan
*to_bam_chan(struct dma_chan
*common
)
386 return container_of(common
, struct bam_chan
, vc
.chan
);
392 struct dma_device common
;
393 struct device_dma_parameters dma_parms
;
394 struct bam_chan
*channels
;
398 /* execution environment ID, from DT */
400 bool controlled_remotely
;
402 const struct reg_offset_data
*layout
;
407 /* dma start transaction tasklet */
408 struct tasklet_struct task
;
412 * bam_addr - returns BAM register address
414 * @pipe: pipe instance (ignored when register doesn't have multiple instances)
415 * @reg: register enum
417 static inline void __iomem
*bam_addr(struct bam_device
*bdev
, u32 pipe
,
420 const struct reg_offset_data r
= bdev
->layout
[reg
];
422 return bdev
->regs
+ r
.base_offset
+
425 r
.ee_mult
* bdev
->ee
;
429 * bam_reset_channel - Reset individual BAM DMA channel
430 * @bchan: bam channel
432 * This function resets a specific BAM channel
434 static void bam_reset_channel(struct bam_chan
*bchan
)
436 struct bam_device
*bdev
= bchan
->bdev
;
438 lockdep_assert_held(&bchan
->vc
.lock
);
441 writel_relaxed(1, bam_addr(bdev
, bchan
->id
, BAM_P_RST
));
442 writel_relaxed(0, bam_addr(bdev
, bchan
->id
, BAM_P_RST
));
444 /* don't allow cpu to reorder BAM register accesses done after this */
447 /* make sure hw is initialized when channel is used the first time */
448 bchan
->initialized
= 0;
452 * bam_chan_init_hw - Initialize channel hardware
453 * @bchan: bam channel
455 * This function resets and initializes the BAM channel
457 static void bam_chan_init_hw(struct bam_chan
*bchan
,
458 enum dma_transfer_direction dir
)
460 struct bam_device
*bdev
= bchan
->bdev
;
463 /* Reset the channel to clear internal state of the FIFO */
464 bam_reset_channel(bchan
);
467 * write out 8 byte aligned address. We have enough space for this
468 * because we allocated 1 more descriptor (8 bytes) than we can use
470 writel_relaxed(ALIGN(bchan
->fifo_phys
, sizeof(struct bam_desc_hw
)),
471 bam_addr(bdev
, bchan
->id
, BAM_P_DESC_FIFO_ADDR
));
472 writel_relaxed(BAM_FIFO_SIZE
,
473 bam_addr(bdev
, bchan
->id
, BAM_P_FIFO_SIZES
));
475 /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */
476 writel_relaxed(P_DEFAULT_IRQS_EN
,
477 bam_addr(bdev
, bchan
->id
, BAM_P_IRQ_EN
));
479 /* unmask the specific pipe and EE combo */
480 val
= readl_relaxed(bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
481 val
|= BIT(bchan
->id
);
482 writel_relaxed(val
, bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
484 /* don't allow cpu to reorder the channel enable done below */
487 /* set fixed direction and mode, then enable channel */
488 val
= P_EN
| P_SYS_MODE
;
489 if (dir
== DMA_DEV_TO_MEM
)
492 writel_relaxed(val
, bam_addr(bdev
, bchan
->id
, BAM_P_CTRL
));
494 bchan
->initialized
= 1;
496 /* init FIFO pointers */
502 * bam_alloc_chan - Allocate channel resources for DMA channel.
503 * @chan: specified channel
505 * This function allocates the FIFO descriptor memory
507 static int bam_alloc_chan(struct dma_chan
*chan
)
509 struct bam_chan
*bchan
= to_bam_chan(chan
);
510 struct bam_device
*bdev
= bchan
->bdev
;
512 if (bchan
->fifo_virt
)
515 /* allocate FIFO descriptor space, but only if necessary */
516 bchan
->fifo_virt
= dma_alloc_wc(bdev
->dev
, BAM_DESC_FIFO_SIZE
,
517 &bchan
->fifo_phys
, GFP_KERNEL
);
519 if (!bchan
->fifo_virt
) {
520 dev_err(bdev
->dev
, "Failed to allocate desc fifo\n");
528 * bam_free_chan - Frees dma resources associated with specific channel
529 * @chan: specified channel
531 * Free the allocated fifo descriptor memory and channel resources
534 static void bam_free_chan(struct dma_chan
*chan
)
536 struct bam_chan
*bchan
= to_bam_chan(chan
);
537 struct bam_device
*bdev
= bchan
->bdev
;
542 ret
= pm_runtime_get_sync(bdev
->dev
);
546 vchan_free_chan_resources(to_virt_chan(chan
));
548 if (!list_empty(&bchan
->desc_list
)) {
549 dev_err(bchan
->bdev
->dev
, "Cannot free busy channel\n");
553 spin_lock_irqsave(&bchan
->vc
.lock
, flags
);
554 bam_reset_channel(bchan
);
555 spin_unlock_irqrestore(&bchan
->vc
.lock
, flags
);
557 dma_free_wc(bdev
->dev
, BAM_DESC_FIFO_SIZE
, bchan
->fifo_virt
,
559 bchan
->fifo_virt
= NULL
;
561 /* mask irq for pipe/channel */
562 val
= readl_relaxed(bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
563 val
&= ~BIT(bchan
->id
);
564 writel_relaxed(val
, bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
567 writel_relaxed(0, bam_addr(bdev
, bchan
->id
, BAM_P_IRQ_EN
));
570 pm_runtime_mark_last_busy(bdev
->dev
);
571 pm_runtime_put_autosuspend(bdev
->dev
);
575 * bam_slave_config - set slave configuration for channel
577 * @cfg: slave configuration
579 * Sets slave configuration for channel
582 static int bam_slave_config(struct dma_chan
*chan
,
583 struct dma_slave_config
*cfg
)
585 struct bam_chan
*bchan
= to_bam_chan(chan
);
588 spin_lock_irqsave(&bchan
->vc
.lock
, flag
);
589 memcpy(&bchan
->slave
, cfg
, sizeof(*cfg
));
590 bchan
->reconfigure
= 1;
591 spin_unlock_irqrestore(&bchan
->vc
.lock
, flag
);
597 * bam_prep_slave_sg - Prep slave sg transaction
600 * @sgl: scatter gather list
601 * @sg_len: length of sg
602 * @direction: DMA transfer direction
604 * @context: transfer context (unused)
606 static struct dma_async_tx_descriptor
*bam_prep_slave_sg(struct dma_chan
*chan
,
607 struct scatterlist
*sgl
, unsigned int sg_len
,
608 enum dma_transfer_direction direction
, unsigned long flags
,
611 struct bam_chan
*bchan
= to_bam_chan(chan
);
612 struct bam_device
*bdev
= bchan
->bdev
;
613 struct bam_async_desc
*async_desc
;
614 struct scatterlist
*sg
;
616 struct bam_desc_hw
*desc
;
617 unsigned int num_alloc
= 0;
620 if (!is_slave_direction(direction
)) {
621 dev_err(bdev
->dev
, "invalid dma direction\n");
625 /* calculate number of required entries */
626 for_each_sg(sgl
, sg
, sg_len
, i
)
627 num_alloc
+= DIV_ROUND_UP(sg_dma_len(sg
), BAM_FIFO_SIZE
);
629 /* allocate enough room to accomodate the number of entries */
630 async_desc
= kzalloc(sizeof(*async_desc
) +
631 (num_alloc
* sizeof(struct bam_desc_hw
)), GFP_NOWAIT
);
636 if (flags
& DMA_PREP_FENCE
)
637 async_desc
->flags
|= DESC_FLAG_NWD
;
639 if (flags
& DMA_PREP_INTERRUPT
)
640 async_desc
->flags
|= DESC_FLAG_EOT
;
642 async_desc
->num_desc
= num_alloc
;
643 async_desc
->curr_desc
= async_desc
->desc
;
644 async_desc
->dir
= direction
;
646 /* fill in temporary descriptors */
647 desc
= async_desc
->desc
;
648 for_each_sg(sgl
, sg
, sg_len
, i
) {
649 unsigned int remainder
= sg_dma_len(sg
);
650 unsigned int curr_offset
= 0;
653 if (flags
& DMA_PREP_CMD
)
654 desc
->flags
|= cpu_to_le16(DESC_FLAG_CMD
);
656 desc
->addr
= cpu_to_le32(sg_dma_address(sg
) +
659 if (remainder
> BAM_FIFO_SIZE
) {
660 desc
->size
= cpu_to_le16(BAM_FIFO_SIZE
);
661 remainder
-= BAM_FIFO_SIZE
;
662 curr_offset
+= BAM_FIFO_SIZE
;
664 desc
->size
= cpu_to_le16(remainder
);
668 async_desc
->length
+= desc
->size
;
670 } while (remainder
> 0);
673 return vchan_tx_prep(&bchan
->vc
, &async_desc
->vd
, flags
);
681 * bam_dma_terminate_all - terminate all transactions on a channel
682 * @bchan: bam dma channel
684 * Dequeues and frees all transactions
685 * No callbacks are done
688 static int bam_dma_terminate_all(struct dma_chan
*chan
)
690 struct bam_chan
*bchan
= to_bam_chan(chan
);
691 struct bam_async_desc
*async_desc
, *tmp
;
695 /* remove all transactions, including active transaction */
696 spin_lock_irqsave(&bchan
->vc
.lock
, flag
);
697 list_for_each_entry_safe(async_desc
, tmp
,
698 &bchan
->desc_list
, desc_node
) {
699 list_add(&async_desc
->vd
.node
, &bchan
->vc
.desc_issued
);
700 list_del(&async_desc
->desc_node
);
703 vchan_get_all_descriptors(&bchan
->vc
, &head
);
704 spin_unlock_irqrestore(&bchan
->vc
.lock
, flag
);
706 vchan_dma_desc_free_list(&bchan
->vc
, &head
);
712 * bam_pause - Pause DMA channel
716 static int bam_pause(struct dma_chan
*chan
)
718 struct bam_chan
*bchan
= to_bam_chan(chan
);
719 struct bam_device
*bdev
= bchan
->bdev
;
723 ret
= pm_runtime_get_sync(bdev
->dev
);
727 spin_lock_irqsave(&bchan
->vc
.lock
, flag
);
728 writel_relaxed(1, bam_addr(bdev
, bchan
->id
, BAM_P_HALT
));
730 spin_unlock_irqrestore(&bchan
->vc
.lock
, flag
);
731 pm_runtime_mark_last_busy(bdev
->dev
);
732 pm_runtime_put_autosuspend(bdev
->dev
);
738 * bam_resume - Resume DMA channel operations
742 static int bam_resume(struct dma_chan
*chan
)
744 struct bam_chan
*bchan
= to_bam_chan(chan
);
745 struct bam_device
*bdev
= bchan
->bdev
;
749 ret
= pm_runtime_get_sync(bdev
->dev
);
753 spin_lock_irqsave(&bchan
->vc
.lock
, flag
);
754 writel_relaxed(0, bam_addr(bdev
, bchan
->id
, BAM_P_HALT
));
756 spin_unlock_irqrestore(&bchan
->vc
.lock
, flag
);
757 pm_runtime_mark_last_busy(bdev
->dev
);
758 pm_runtime_put_autosuspend(bdev
->dev
);
764 * process_channel_irqs - processes the channel interrupts
765 * @bdev: bam controller
767 * This function processes the channel interrupts
770 static u32
process_channel_irqs(struct bam_device
*bdev
)
772 u32 i
, srcs
, pipe_stts
, offset
, avail
;
774 struct bam_async_desc
*async_desc
, *tmp
;
776 srcs
= readl_relaxed(bam_addr(bdev
, 0, BAM_IRQ_SRCS_EE
));
778 /* return early if no pipe/channel interrupts are present */
782 for (i
= 0; i
< bdev
->num_channels
; i
++) {
783 struct bam_chan
*bchan
= &bdev
->channels
[i
];
785 if (!(srcs
& BIT(i
)))
789 pipe_stts
= readl_relaxed(bam_addr(bdev
, i
, BAM_P_IRQ_STTS
));
791 writel_relaxed(pipe_stts
, bam_addr(bdev
, i
, BAM_P_IRQ_CLR
));
793 spin_lock_irqsave(&bchan
->vc
.lock
, flags
);
795 offset
= readl_relaxed(bam_addr(bdev
, i
, BAM_P_SW_OFSTS
)) &
797 offset
/= sizeof(struct bam_desc_hw
);
799 /* Number of bytes available to read */
800 avail
= CIRC_CNT(offset
, bchan
->head
, MAX_DESCRIPTORS
+ 1);
802 list_for_each_entry_safe(async_desc
, tmp
,
803 &bchan
->desc_list
, desc_node
) {
804 /* Not enough data to read */
805 if (avail
< async_desc
->xfer_len
)
809 bchan
->head
+= async_desc
->xfer_len
;
810 bchan
->head
%= MAX_DESCRIPTORS
;
812 async_desc
->num_desc
-= async_desc
->xfer_len
;
813 async_desc
->curr_desc
+= async_desc
->xfer_len
;
814 avail
-= async_desc
->xfer_len
;
817 * if complete, process cookie. Otherwise
818 * push back to front of desc_issued so that
819 * it gets restarted by the tasklet
821 if (!async_desc
->num_desc
) {
822 vchan_cookie_complete(&async_desc
->vd
);
824 list_add(&async_desc
->vd
.node
,
825 &bchan
->vc
.desc_issued
);
827 list_del(&async_desc
->desc_node
);
830 spin_unlock_irqrestore(&bchan
->vc
.lock
, flags
);
837 * bam_dma_irq - irq handler for bam controller
838 * @irq: IRQ of interrupt
839 * @data: callback data
841 * IRQ handler for the bam controller
843 static irqreturn_t
bam_dma_irq(int irq
, void *data
)
845 struct bam_device
*bdev
= data
;
846 u32 clr_mask
= 0, srcs
= 0;
849 srcs
|= process_channel_irqs(bdev
);
851 /* kick off tasklet to start next dma transfer */
853 tasklet_schedule(&bdev
->task
);
855 ret
= pm_runtime_get_sync(bdev
->dev
);
859 if (srcs
& BAM_IRQ
) {
860 clr_mask
= readl_relaxed(bam_addr(bdev
, 0, BAM_IRQ_STTS
));
863 * don't allow reorder of the various accesses to the BAM
868 writel_relaxed(clr_mask
, bam_addr(bdev
, 0, BAM_IRQ_CLR
));
871 pm_runtime_mark_last_busy(bdev
->dev
);
872 pm_runtime_put_autosuspend(bdev
->dev
);
878 * bam_tx_status - returns status of transaction
880 * @cookie: transaction cookie
881 * @txstate: DMA transaction state
883 * Return status of dma transaction
885 static enum dma_status
bam_tx_status(struct dma_chan
*chan
, dma_cookie_t cookie
,
886 struct dma_tx_state
*txstate
)
888 struct bam_chan
*bchan
= to_bam_chan(chan
);
889 struct bam_async_desc
*async_desc
;
890 struct virt_dma_desc
*vd
;
896 ret
= dma_cookie_status(chan
, cookie
, txstate
);
897 if (ret
== DMA_COMPLETE
)
901 return bchan
->paused
? DMA_PAUSED
: ret
;
903 spin_lock_irqsave(&bchan
->vc
.lock
, flags
);
904 vd
= vchan_find_desc(&bchan
->vc
, cookie
);
906 residue
= container_of(vd
, struct bam_async_desc
, vd
)->length
;
908 list_for_each_entry(async_desc
, &bchan
->desc_list
, desc_node
) {
909 if (async_desc
->vd
.tx
.cookie
!= cookie
)
912 for (i
= 0; i
< async_desc
->num_desc
; i
++)
913 residue
+= async_desc
->curr_desc
[i
].size
;
917 spin_unlock_irqrestore(&bchan
->vc
.lock
, flags
);
919 dma_set_residue(txstate
, residue
);
921 if (ret
== DMA_IN_PROGRESS
&& bchan
->paused
)
928 * bam_apply_new_config
929 * @bchan: bam dma channel
930 * @dir: DMA direction
932 static void bam_apply_new_config(struct bam_chan
*bchan
,
933 enum dma_transfer_direction dir
)
935 struct bam_device
*bdev
= bchan
->bdev
;
938 if (dir
== DMA_DEV_TO_MEM
)
939 maxburst
= bchan
->slave
.src_maxburst
;
941 maxburst
= bchan
->slave
.dst_maxburst
;
943 writel_relaxed(maxburst
, bam_addr(bdev
, 0, BAM_DESC_CNT_TRSHLD
));
945 bchan
->reconfigure
= 0;
949 * bam_start_dma - start next transaction
950 * @bchan - bam dma channel
952 static void bam_start_dma(struct bam_chan
*bchan
)
954 struct virt_dma_desc
*vd
= vchan_next_desc(&bchan
->vc
);
955 struct bam_device
*bdev
= bchan
->bdev
;
956 struct bam_async_desc
*async_desc
= NULL
;
957 struct bam_desc_hw
*desc
;
958 struct bam_desc_hw
*fifo
= PTR_ALIGN(bchan
->fifo_virt
,
959 sizeof(struct bam_desc_hw
));
962 struct dmaengine_desc_callback cb
;
964 lockdep_assert_held(&bchan
->vc
.lock
);
969 ret
= pm_runtime_get_sync(bdev
->dev
);
973 while (vd
&& !IS_BUSY(bchan
)) {
976 async_desc
= container_of(vd
, struct bam_async_desc
, vd
);
978 /* on first use, initialize the channel hardware */
979 if (!bchan
->initialized
)
980 bam_chan_init_hw(bchan
, async_desc
->dir
);
982 /* apply new slave config changes, if necessary */
983 if (bchan
->reconfigure
)
984 bam_apply_new_config(bchan
, async_desc
->dir
);
986 desc
= async_desc
->curr_desc
;
987 avail
= CIRC_SPACE(bchan
->tail
, bchan
->head
,
988 MAX_DESCRIPTORS
+ 1);
990 if (async_desc
->num_desc
> avail
)
991 async_desc
->xfer_len
= avail
;
993 async_desc
->xfer_len
= async_desc
->num_desc
;
995 /* set any special flags on the last descriptor */
996 if (async_desc
->num_desc
== async_desc
->xfer_len
)
997 desc
[async_desc
->xfer_len
- 1].flags
|=
998 cpu_to_le16(async_desc
->flags
);
1000 vd
= vchan_next_desc(&bchan
->vc
);
1002 dmaengine_desc_get_callback(&async_desc
->vd
.tx
, &cb
);
1005 * An interrupt is generated at this desc, if
1007 * - No more descriptors to add.
1008 * - If a callback completion was requested for this DESC,
1009 * In this case, BAM will deliver the completion callback
1010 * for this desc and continue processing the next desc.
1012 if (((avail
<= async_desc
->xfer_len
) || !vd
||
1013 dmaengine_desc_callback_valid(&cb
)) &&
1014 !(async_desc
->flags
& DESC_FLAG_EOT
))
1015 desc
[async_desc
->xfer_len
- 1].flags
|=
1016 cpu_to_le16(DESC_FLAG_INT
);
1018 if (bchan
->tail
+ async_desc
->xfer_len
> MAX_DESCRIPTORS
) {
1019 u32 partial
= MAX_DESCRIPTORS
- bchan
->tail
;
1021 memcpy(&fifo
[bchan
->tail
], desc
,
1022 partial
* sizeof(struct bam_desc_hw
));
1023 memcpy(fifo
, &desc
[partial
],
1024 (async_desc
->xfer_len
- partial
) *
1025 sizeof(struct bam_desc_hw
));
1027 memcpy(&fifo
[bchan
->tail
], desc
,
1028 async_desc
->xfer_len
*
1029 sizeof(struct bam_desc_hw
));
1032 bchan
->tail
+= async_desc
->xfer_len
;
1033 bchan
->tail
%= MAX_DESCRIPTORS
;
1034 list_add_tail(&async_desc
->desc_node
, &bchan
->desc_list
);
1037 /* ensure descriptor writes and dma start not reordered */
1039 writel_relaxed(bchan
->tail
* sizeof(struct bam_desc_hw
),
1040 bam_addr(bdev
, bchan
->id
, BAM_P_EVNT_REG
));
1042 pm_runtime_mark_last_busy(bdev
->dev
);
1043 pm_runtime_put_autosuspend(bdev
->dev
);
1047 * dma_tasklet - DMA IRQ tasklet
1048 * @data: tasklet argument (bam controller structure)
1050 * Sets up next DMA operation and then processes all completed transactions
1052 static void dma_tasklet(unsigned long data
)
1054 struct bam_device
*bdev
= (struct bam_device
*)data
;
1055 struct bam_chan
*bchan
;
1056 unsigned long flags
;
1059 /* go through the channels and kick off transactions */
1060 for (i
= 0; i
< bdev
->num_channels
; i
++) {
1061 bchan
= &bdev
->channels
[i
];
1062 spin_lock_irqsave(&bchan
->vc
.lock
, flags
);
1064 if (!list_empty(&bchan
->vc
.desc_issued
) && !IS_BUSY(bchan
))
1065 bam_start_dma(bchan
);
1066 spin_unlock_irqrestore(&bchan
->vc
.lock
, flags
);
1072 * bam_issue_pending - starts pending transactions
1073 * @chan: dma channel
1075 * Calls tasklet directly which in turn starts any pending transactions
1077 static void bam_issue_pending(struct dma_chan
*chan
)
1079 struct bam_chan
*bchan
= to_bam_chan(chan
);
1080 unsigned long flags
;
1082 spin_lock_irqsave(&bchan
->vc
.lock
, flags
);
1084 /* if work pending and idle, start a transaction */
1085 if (vchan_issue_pending(&bchan
->vc
) && !IS_BUSY(bchan
))
1086 bam_start_dma(bchan
);
1088 spin_unlock_irqrestore(&bchan
->vc
.lock
, flags
);
1092 * bam_dma_free_desc - free descriptor memory
1093 * @vd: virtual descriptor
1096 static void bam_dma_free_desc(struct virt_dma_desc
*vd
)
1098 struct bam_async_desc
*async_desc
= container_of(vd
,
1099 struct bam_async_desc
, vd
);
1104 static struct dma_chan
*bam_dma_xlate(struct of_phandle_args
*dma_spec
,
1107 struct bam_device
*bdev
= container_of(of
->of_dma_data
,
1108 struct bam_device
, common
);
1109 unsigned int request
;
1111 if (dma_spec
->args_count
!= 1)
1114 request
= dma_spec
->args
[0];
1115 if (request
>= bdev
->num_channels
)
1118 return dma_get_slave_channel(&(bdev
->channels
[request
].vc
.chan
));
1125 * Initialization helper for global bam registers
1127 static int bam_init(struct bam_device
*bdev
)
1131 /* read revision and configuration information */
1132 if (!bdev
->num_ees
) {
1133 val
= readl_relaxed(bam_addr(bdev
, 0, BAM_REVISION
));
1134 bdev
->num_ees
= (val
>> NUM_EES_SHIFT
) & NUM_EES_MASK
;
1137 /* check that configured EE is within range */
1138 if (bdev
->ee
>= bdev
->num_ees
)
1141 if (!bdev
->num_channels
) {
1142 val
= readl_relaxed(bam_addr(bdev
, 0, BAM_NUM_PIPES
));
1143 bdev
->num_channels
= val
& BAM_NUM_PIPES_MASK
;
1146 if (bdev
->controlled_remotely
)
1150 /* after reset all pipes are disabled and idle */
1151 val
= readl_relaxed(bam_addr(bdev
, 0, BAM_CTRL
));
1153 writel_relaxed(val
, bam_addr(bdev
, 0, BAM_CTRL
));
1155 writel_relaxed(val
, bam_addr(bdev
, 0, BAM_CTRL
));
1157 /* make sure previous stores are visible before enabling BAM */
1162 writel_relaxed(val
, bam_addr(bdev
, 0, BAM_CTRL
));
1164 /* set descriptor threshhold, start with 4 bytes */
1165 writel_relaxed(DEFAULT_CNT_THRSHLD
,
1166 bam_addr(bdev
, 0, BAM_DESC_CNT_TRSHLD
));
1168 /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
1169 writel_relaxed(BAM_CNFG_BITS_DEFAULT
, bam_addr(bdev
, 0, BAM_CNFG_BITS
));
1171 /* enable irqs for errors */
1172 writel_relaxed(BAM_ERROR_EN
| BAM_HRESP_ERR_EN
,
1173 bam_addr(bdev
, 0, BAM_IRQ_EN
));
1175 /* unmask global bam interrupt */
1176 writel_relaxed(BAM_IRQ_MSK
, bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
1181 static void bam_channel_init(struct bam_device
*bdev
, struct bam_chan
*bchan
,
1187 vchan_init(&bchan
->vc
, &bdev
->common
);
1188 bchan
->vc
.desc_free
= bam_dma_free_desc
;
1189 INIT_LIST_HEAD(&bchan
->desc_list
);
1192 static const struct of_device_id bam_of_match
[] = {
1193 { .compatible
= "qcom,bam-v1.3.0", .data
= &bam_v1_3_reg_info
},
1194 { .compatible
= "qcom,bam-v1.4.0", .data
= &bam_v1_4_reg_info
},
1195 { .compatible
= "qcom,bam-v1.7.0", .data
= &bam_v1_7_reg_info
},
1199 MODULE_DEVICE_TABLE(of
, bam_of_match
);
1201 static int bam_dma_probe(struct platform_device
*pdev
)
1203 struct bam_device
*bdev
;
1204 const struct of_device_id
*match
;
1205 struct resource
*iores
;
1208 bdev
= devm_kzalloc(&pdev
->dev
, sizeof(*bdev
), GFP_KERNEL
);
1212 bdev
->dev
= &pdev
->dev
;
1214 match
= of_match_node(bam_of_match
, pdev
->dev
.of_node
);
1216 dev_err(&pdev
->dev
, "Unsupported BAM module\n");
1220 bdev
->layout
= match
->data
;
1222 iores
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1223 bdev
->regs
= devm_ioremap_resource(&pdev
->dev
, iores
);
1224 if (IS_ERR(bdev
->regs
))
1225 return PTR_ERR(bdev
->regs
);
1227 bdev
->irq
= platform_get_irq(pdev
, 0);
1231 ret
= of_property_read_u32(pdev
->dev
.of_node
, "qcom,ee", &bdev
->ee
);
1233 dev_err(bdev
->dev
, "Execution environment unspecified\n");
1237 bdev
->controlled_remotely
= of_property_read_bool(pdev
->dev
.of_node
,
1238 "qcom,controlled-remotely");
1240 if (bdev
->controlled_remotely
) {
1241 ret
= of_property_read_u32(pdev
->dev
.of_node
, "num-channels",
1242 &bdev
->num_channels
);
1244 dev_err(bdev
->dev
, "num-channels unspecified in dt\n");
1246 ret
= of_property_read_u32(pdev
->dev
.of_node
, "qcom,num-ees",
1249 dev_err(bdev
->dev
, "num-ees unspecified in dt\n");
1252 bdev
->bamclk
= devm_clk_get(bdev
->dev
, "bam_clk");
1253 if (IS_ERR(bdev
->bamclk
))
1254 return PTR_ERR(bdev
->bamclk
);
1256 ret
= clk_prepare_enable(bdev
->bamclk
);
1258 dev_err(bdev
->dev
, "failed to prepare/enable clock\n");
1262 ret
= bam_init(bdev
);
1264 goto err_disable_clk
;
1266 tasklet_init(&bdev
->task
, dma_tasklet
, (unsigned long)bdev
);
1268 bdev
->channels
= devm_kcalloc(bdev
->dev
, bdev
->num_channels
,
1269 sizeof(*bdev
->channels
), GFP_KERNEL
);
1271 if (!bdev
->channels
) {
1273 goto err_tasklet_kill
;
1276 /* allocate and initialize channels */
1277 INIT_LIST_HEAD(&bdev
->common
.channels
);
1279 for (i
= 0; i
< bdev
->num_channels
; i
++)
1280 bam_channel_init(bdev
, &bdev
->channels
[i
], i
);
1282 ret
= devm_request_irq(bdev
->dev
, bdev
->irq
, bam_dma_irq
,
1283 IRQF_TRIGGER_HIGH
, "bam_dma", bdev
);
1285 goto err_bam_channel_exit
;
1287 /* set max dma segment size */
1288 bdev
->common
.dev
= bdev
->dev
;
1289 bdev
->common
.dev
->dma_parms
= &bdev
->dma_parms
;
1290 ret
= dma_set_max_seg_size(bdev
->common
.dev
, BAM_FIFO_SIZE
);
1292 dev_err(bdev
->dev
, "cannot set maximum segment size\n");
1293 goto err_bam_channel_exit
;
1296 platform_set_drvdata(pdev
, bdev
);
1298 /* set capabilities */
1299 dma_cap_zero(bdev
->common
.cap_mask
);
1300 dma_cap_set(DMA_SLAVE
, bdev
->common
.cap_mask
);
1302 /* initialize dmaengine apis */
1303 bdev
->common
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
1304 bdev
->common
.residue_granularity
= DMA_RESIDUE_GRANULARITY_SEGMENT
;
1305 bdev
->common
.src_addr_widths
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
1306 bdev
->common
.dst_addr_widths
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
1307 bdev
->common
.device_alloc_chan_resources
= bam_alloc_chan
;
1308 bdev
->common
.device_free_chan_resources
= bam_free_chan
;
1309 bdev
->common
.device_prep_slave_sg
= bam_prep_slave_sg
;
1310 bdev
->common
.device_config
= bam_slave_config
;
1311 bdev
->common
.device_pause
= bam_pause
;
1312 bdev
->common
.device_resume
= bam_resume
;
1313 bdev
->common
.device_terminate_all
= bam_dma_terminate_all
;
1314 bdev
->common
.device_issue_pending
= bam_issue_pending
;
1315 bdev
->common
.device_tx_status
= bam_tx_status
;
1316 bdev
->common
.dev
= bdev
->dev
;
1318 ret
= dma_async_device_register(&bdev
->common
);
1320 dev_err(bdev
->dev
, "failed to register dma async device\n");
1321 goto err_bam_channel_exit
;
1324 ret
= of_dma_controller_register(pdev
->dev
.of_node
, bam_dma_xlate
,
1327 goto err_unregister_dma
;
1329 pm_runtime_irq_safe(&pdev
->dev
);
1330 pm_runtime_set_autosuspend_delay(&pdev
->dev
, BAM_DMA_AUTOSUSPEND_DELAY
);
1331 pm_runtime_use_autosuspend(&pdev
->dev
);
1332 pm_runtime_mark_last_busy(&pdev
->dev
);
1333 pm_runtime_set_active(&pdev
->dev
);
1334 pm_runtime_enable(&pdev
->dev
);
1339 dma_async_device_unregister(&bdev
->common
);
1340 err_bam_channel_exit
:
1341 for (i
= 0; i
< bdev
->num_channels
; i
++)
1342 tasklet_kill(&bdev
->channels
[i
].vc
.task
);
1344 tasklet_kill(&bdev
->task
);
1346 clk_disable_unprepare(bdev
->bamclk
);
1351 static int bam_dma_remove(struct platform_device
*pdev
)
1353 struct bam_device
*bdev
= platform_get_drvdata(pdev
);
1356 pm_runtime_force_suspend(&pdev
->dev
);
1358 of_dma_controller_free(pdev
->dev
.of_node
);
1359 dma_async_device_unregister(&bdev
->common
);
1361 /* mask all interrupts for this execution environment */
1362 writel_relaxed(0, bam_addr(bdev
, 0, BAM_IRQ_SRCS_MSK_EE
));
1364 devm_free_irq(bdev
->dev
, bdev
->irq
, bdev
);
1366 for (i
= 0; i
< bdev
->num_channels
; i
++) {
1367 bam_dma_terminate_all(&bdev
->channels
[i
].vc
.chan
);
1368 tasklet_kill(&bdev
->channels
[i
].vc
.task
);
1370 if (!bdev
->channels
[i
].fifo_virt
)
1373 dma_free_wc(bdev
->dev
, BAM_DESC_FIFO_SIZE
,
1374 bdev
->channels
[i
].fifo_virt
,
1375 bdev
->channels
[i
].fifo_phys
);
1378 tasklet_kill(&bdev
->task
);
1380 clk_disable_unprepare(bdev
->bamclk
);
1385 static int __maybe_unused
bam_dma_runtime_suspend(struct device
*dev
)
1387 struct bam_device
*bdev
= dev_get_drvdata(dev
);
1389 clk_disable(bdev
->bamclk
);
1394 static int __maybe_unused
bam_dma_runtime_resume(struct device
*dev
)
1396 struct bam_device
*bdev
= dev_get_drvdata(dev
);
1399 ret
= clk_enable(bdev
->bamclk
);
1401 dev_err(dev
, "clk_enable failed: %d\n", ret
);
1408 static int __maybe_unused
bam_dma_suspend(struct device
*dev
)
1410 struct bam_device
*bdev
= dev_get_drvdata(dev
);
1412 pm_runtime_force_suspend(dev
);
1414 clk_unprepare(bdev
->bamclk
);
1419 static int __maybe_unused
bam_dma_resume(struct device
*dev
)
1421 struct bam_device
*bdev
= dev_get_drvdata(dev
);
1424 ret
= clk_prepare(bdev
->bamclk
);
1428 pm_runtime_force_resume(dev
);
1433 static const struct dev_pm_ops bam_dma_pm_ops
= {
1434 SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend
, bam_dma_resume
)
1435 SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend
, bam_dma_runtime_resume
,
1439 static struct platform_driver bam_dma_driver
= {
1440 .probe
= bam_dma_probe
,
1441 .remove
= bam_dma_remove
,
1443 .name
= "bam-dma-engine",
1444 .pm
= &bam_dma_pm_ops
,
1445 .of_match_table
= bam_of_match
,
1449 module_platform_driver(bam_dma_driver
);
1451 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1452 MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1453 MODULE_LICENSE("GPL v2");