3 * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
5 * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/init.h>
13 #include <linux/platform_device.h>
14 #include <linux/err.h>
15 #include <linux/spinlock.h>
16 #include <linux/delay.h>
17 #include <linux/list.h>
18 #include <linux/clk.h>
19 #include <linux/vmalloc.h>
20 #include <linux/string.h>
21 #include <linux/interrupt.h>
26 #include "ipu_intern.h"
28 #define FS_VF_IN_VALID 0x00000002
29 #define FS_ENC_IN_VALID 0x00000001
32 * There can be only one, we could allocate it dynamically, but then we'd have
33 * to add an extra parameter to some functions, and use something as ugly as
34 * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
37 static struct ipu ipu_data
;
39 #define to_ipu(id) container_of(id, struct ipu, idmac)
41 static u32
__idmac_read_icreg(struct ipu
*ipu
, unsigned long reg
)
43 return __raw_readl(ipu
->reg_ic
+ reg
);
46 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
48 static void __idmac_write_icreg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
50 __raw_writel(value
, ipu
->reg_ic
+ reg
);
53 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
55 static u32
idmac_read_ipureg(struct ipu
*ipu
, unsigned long reg
)
57 return __raw_readl(ipu
->reg_ipu
+ reg
);
60 static void idmac_write_ipureg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
62 __raw_writel(value
, ipu
->reg_ipu
+ reg
);
65 /*****************************************************************************
66 * IPU / IC common functions
68 static void dump_idmac_reg(struct ipu
*ipu
)
70 dev_dbg(ipu
->dev
, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
71 "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
72 idmac_read_icreg(ipu
, IDMAC_CONF
),
73 idmac_read_icreg(ipu
, IC_CONF
),
74 idmac_read_icreg(ipu
, IDMAC_CHA_EN
),
75 idmac_read_icreg(ipu
, IDMAC_CHA_PRI
),
76 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
));
77 dev_dbg(ipu
->dev
, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
78 "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
79 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
80 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
81 idmac_read_ipureg(ipu
, IPU_CHA_CUR_BUF
),
82 idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
),
83 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
86 static uint32_t bytes_per_pixel(enum pixel_fmt fmt
)
89 case IPU_PIX_FMT_GENERIC
: /* generic data */
90 case IPU_PIX_FMT_RGB332
:
91 case IPU_PIX_FMT_YUV420P
:
92 case IPU_PIX_FMT_YUV422P
:
95 case IPU_PIX_FMT_RGB565
:
96 case IPU_PIX_FMT_YUYV
:
97 case IPU_PIX_FMT_UYVY
:
99 case IPU_PIX_FMT_BGR24
:
100 case IPU_PIX_FMT_RGB24
:
102 case IPU_PIX_FMT_GENERIC_32
: /* generic data */
103 case IPU_PIX_FMT_BGR32
:
104 case IPU_PIX_FMT_RGB32
:
105 case IPU_PIX_FMT_ABGR32
:
110 /* Enable / disable direct write to memory by the Camera Sensor Interface */
111 static void ipu_ic_enable_task(struct ipu
*ipu
, enum ipu_channel channel
)
113 uint32_t ic_conf
, mask
;
117 mask
= IC_CONF_PRPENC_EN
;
120 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
125 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) | mask
;
126 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
129 static void ipu_ic_disable_task(struct ipu
*ipu
, enum ipu_channel channel
)
131 uint32_t ic_conf
, mask
;
135 mask
= IC_CONF_PRPENC_EN
;
138 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
143 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) & ~mask
;
144 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
147 static uint32_t ipu_channel_status(struct ipu
*ipu
, enum ipu_channel channel
)
149 uint32_t stat
= TASK_STAT_IDLE
;
150 uint32_t task_stat_reg
= idmac_read_ipureg(ipu
, IPU_TASKS_STAT
);
154 stat
= (task_stat_reg
& TSTAT_CSI2MEM_MASK
) >>
155 TSTAT_CSI2MEM_OFFSET
;
166 struct chan_param_mem_planar
{
204 } __attribute__ ((packed
));
206 struct chan_param_mem_interleaved
{
263 } __attribute__ ((packed
));
265 union chan_param_mem
{
266 struct chan_param_mem_planar pp
;
267 struct chan_param_mem_interleaved ip
;
270 static void ipu_ch_param_set_plane_offset(union chan_param_mem
*params
,
271 u32 u_offset
, u32 v_offset
)
273 params
->pp
.ubo_l
= u_offset
& 0x7ff;
274 params
->pp
.ubo_h
= u_offset
>> 11;
275 params
->pp
.vbo_l
= v_offset
& 0x1ffff;
276 params
->pp
.vbo_h
= v_offset
>> 17;
279 static void ipu_ch_param_set_size(union chan_param_mem
*params
,
280 uint32_t pixel_fmt
, uint16_t width
,
281 uint16_t height
, uint16_t stride
)
286 params
->pp
.fw
= width
- 1;
287 params
->pp
.fh_l
= height
- 1;
288 params
->pp
.fh_h
= (height
- 1) >> 8;
289 params
->pp
.sl
= stride
- 1;
292 case IPU_PIX_FMT_GENERIC
:
293 /*Represents 8-bit Generic data */
297 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
299 case IPU_PIX_FMT_GENERIC_32
:
300 /*Represents 32-bit Generic data */
304 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
306 case IPU_PIX_FMT_RGB565
:
310 params
->ip
.sat
= 2; /* SAT = 32-bit access */
311 params
->ip
.ofs0
= 0; /* Red bit offset */
312 params
->ip
.ofs1
= 5; /* Green bit offset */
313 params
->ip
.ofs2
= 11; /* Blue bit offset */
314 params
->ip
.ofs3
= 16; /* Alpha bit offset */
315 params
->ip
.wid0
= 4; /* Red bit width - 1 */
316 params
->ip
.wid1
= 5; /* Green bit width - 1 */
317 params
->ip
.wid2
= 4; /* Blue bit width - 1 */
319 case IPU_PIX_FMT_BGR24
:
320 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
323 params
->ip
.sat
= 2; /* SAT = 32-bit access */
324 params
->ip
.ofs0
= 0; /* Red bit offset */
325 params
->ip
.ofs1
= 8; /* Green bit offset */
326 params
->ip
.ofs2
= 16; /* Blue bit offset */
327 params
->ip
.ofs3
= 24; /* Alpha bit offset */
328 params
->ip
.wid0
= 7; /* Red bit width - 1 */
329 params
->ip
.wid1
= 7; /* Green bit width - 1 */
330 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
332 case IPU_PIX_FMT_RGB24
:
333 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
336 params
->ip
.sat
= 2; /* SAT = 32-bit access */
337 params
->ip
.ofs0
= 16; /* Red bit offset */
338 params
->ip
.ofs1
= 8; /* Green bit offset */
339 params
->ip
.ofs2
= 0; /* Blue bit offset */
340 params
->ip
.ofs3
= 24; /* Alpha bit offset */
341 params
->ip
.wid0
= 7; /* Red bit width - 1 */
342 params
->ip
.wid1
= 7; /* Green bit width - 1 */
343 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
345 case IPU_PIX_FMT_BGRA32
:
346 case IPU_PIX_FMT_BGR32
:
350 params
->ip
.sat
= 2; /* SAT = 32-bit access */
351 params
->ip
.ofs0
= 8; /* Red bit offset */
352 params
->ip
.ofs1
= 16; /* Green bit offset */
353 params
->ip
.ofs2
= 24; /* Blue bit offset */
354 params
->ip
.ofs3
= 0; /* Alpha bit offset */
355 params
->ip
.wid0
= 7; /* Red bit width - 1 */
356 params
->ip
.wid1
= 7; /* Green bit width - 1 */
357 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
358 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
360 case IPU_PIX_FMT_RGBA32
:
361 case IPU_PIX_FMT_RGB32
:
365 params
->ip
.sat
= 2; /* SAT = 32-bit access */
366 params
->ip
.ofs0
= 24; /* Red bit offset */
367 params
->ip
.ofs1
= 16; /* Green bit offset */
368 params
->ip
.ofs2
= 8; /* Blue bit offset */
369 params
->ip
.ofs3
= 0; /* Alpha bit offset */
370 params
->ip
.wid0
= 7; /* Red bit width - 1 */
371 params
->ip
.wid1
= 7; /* Green bit width - 1 */
372 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
373 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
375 case IPU_PIX_FMT_ABGR32
:
379 params
->ip
.sat
= 2; /* SAT = 32-bit access */
380 params
->ip
.ofs0
= 8; /* Red bit offset */
381 params
->ip
.ofs1
= 16; /* Green bit offset */
382 params
->ip
.ofs2
= 24; /* Blue bit offset */
383 params
->ip
.ofs3
= 0; /* Alpha bit offset */
384 params
->ip
.wid0
= 7; /* Red bit width - 1 */
385 params
->ip
.wid1
= 7; /* Green bit width - 1 */
386 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
387 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
389 case IPU_PIX_FMT_UYVY
:
393 params
->ip
.sat
= 2; /* SAT = 32-bit access */
395 case IPU_PIX_FMT_YUV420P2
:
396 case IPU_PIX_FMT_YUV420P
:
400 params
->ip
.sat
= 2; /* SAT = 32-bit access */
401 u_offset
= stride
* height
;
402 v_offset
= u_offset
+ u_offset
/ 4;
403 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
405 case IPU_PIX_FMT_YVU422P
:
409 params
->ip
.sat
= 2; /* SAT = 32-bit access */
410 v_offset
= stride
* height
;
411 u_offset
= v_offset
+ v_offset
/ 2;
412 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
414 case IPU_PIX_FMT_YUV422P
:
418 params
->ip
.sat
= 2; /* SAT = 32-bit access */
419 u_offset
= stride
* height
;
420 v_offset
= u_offset
+ u_offset
/ 2;
421 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
424 dev_err(ipu_data
.dev
,
425 "mxc ipu: unimplemented pixel format %d\n", pixel_fmt
);
432 static void ipu_ch_param_set_burst_size(union chan_param_mem
*params
,
433 uint16_t burst_pixels
)
435 params
->pp
.npb
= burst_pixels
- 1;
438 static void ipu_ch_param_set_buffer(union chan_param_mem
*params
,
439 dma_addr_t buf0
, dma_addr_t buf1
)
441 params
->pp
.eba0
= buf0
;
442 params
->pp
.eba1
= buf1
;
445 static void ipu_ch_param_set_rotation(union chan_param_mem
*params
,
446 enum ipu_rotate_mode rotate
)
448 params
->pp
.bam
= rotate
;
451 static void ipu_write_param_mem(uint32_t addr
, uint32_t *data
,
454 for (; num_words
> 0; num_words
--) {
455 dev_dbg(ipu_data
.dev
,
456 "write param mem - addr = 0x%08X, data = 0x%08X\n",
458 idmac_write_ipureg(&ipu_data
, addr
, IPU_IMA_ADDR
);
459 idmac_write_ipureg(&ipu_data
, *data
++, IPU_IMA_DATA
);
461 if ((addr
& 0x7) == 5) {
462 addr
&= ~0x7; /* set to word 0 */
463 addr
+= 8; /* increment to next row */
468 static int calc_resize_coeffs(uint32_t in_size
, uint32_t out_size
,
469 uint32_t *resize_coeff
,
470 uint32_t *downsize_coeff
)
473 uint32_t temp_downsize
;
475 *resize_coeff
= 1 << 13;
476 *downsize_coeff
= 1 << 13;
478 /* Cannot downsize more than 8:1 */
479 if (out_size
<< 3 < in_size
)
482 /* compute downsizing coefficient */
485 while (temp_size
>= out_size
* 2 && temp_downsize
< 2) {
489 *downsize_coeff
= temp_downsize
;
492 * compute resizing coefficient using the following formula:
493 * resize_coeff = M*(SI -1)/(SO - 1)
494 * where M = 2^13, SI - input size, SO - output size
496 *resize_coeff
= (8192L * (temp_size
- 1)) / (out_size
- 1);
497 if (*resize_coeff
>= 16384L) {
498 dev_err(ipu_data
.dev
, "Warning! Overflow on resize coeff.\n");
499 *resize_coeff
= 0x3FFF;
502 dev_dbg(ipu_data
.dev
, "resizing from %u -> %u pixels, "
503 "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size
, out_size
,
504 *downsize_coeff
, *resize_coeff
>= 8192L ? 1 : 0,
505 ((*resize_coeff
& 0x1FFF) * 10000L) / 8192L, *resize_coeff
);
510 static enum ipu_color_space
format_to_colorspace(enum pixel_fmt fmt
)
513 case IPU_PIX_FMT_RGB565
:
514 case IPU_PIX_FMT_BGR24
:
515 case IPU_PIX_FMT_RGB24
:
516 case IPU_PIX_FMT_BGR32
:
517 case IPU_PIX_FMT_RGB32
:
518 return IPU_COLORSPACE_RGB
;
520 return IPU_COLORSPACE_YCBCR
;
524 static int ipu_ic_init_prpenc(struct ipu
*ipu
,
525 union ipu_channel_param
*params
, bool src_is_csi
)
527 uint32_t reg
, ic_conf
;
528 uint32_t downsize_coeff
, resize_coeff
;
529 enum ipu_color_space in_fmt
, out_fmt
;
531 /* Setup vertical resizing */
532 calc_resize_coeffs(params
->video
.in_height
,
533 params
->video
.out_height
,
534 &resize_coeff
, &downsize_coeff
);
535 reg
= (downsize_coeff
<< 30) | (resize_coeff
<< 16);
537 /* Setup horizontal resizing */
538 calc_resize_coeffs(params
->video
.in_width
,
539 params
->video
.out_width
,
540 &resize_coeff
, &downsize_coeff
);
541 reg
|= (downsize_coeff
<< 14) | resize_coeff
;
543 /* Setup color space conversion */
544 in_fmt
= format_to_colorspace(params
->video
.in_pixel_fmt
);
545 out_fmt
= format_to_colorspace(params
->video
.out_pixel_fmt
);
548 * Colourspace conversion unsupported yet - see _init_csc() in
551 if (in_fmt
!= out_fmt
) {
552 dev_err(ipu
->dev
, "Colourspace conversion unsupported!\n");
556 idmac_write_icreg(ipu
, reg
, IC_PRP_ENC_RSC
);
558 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
);
561 ic_conf
&= ~IC_CONF_RWS_EN
;
563 ic_conf
|= IC_CONF_RWS_EN
;
565 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
570 static uint32_t dma_param_addr(uint32_t dma_ch
)
572 /* Channel Parameter Memory */
573 return 0x10000 | (dma_ch
<< 4);
576 static void ipu_channel_set_priority(struct ipu
*ipu
, enum ipu_channel channel
,
579 u32 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_PRI
);
582 reg
|= 1UL << channel
;
584 reg
&= ~(1UL << channel
);
586 idmac_write_icreg(ipu
, reg
, IDMAC_CHA_PRI
);
591 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel
)
598 mask
= IPU_CONF_CSI_EN
| IPU_CONF_IC_EN
;
602 mask
= IPU_CONF_SDC_EN
| IPU_CONF_DI_EN
;
613 * ipu_enable_channel() - enable an IPU channel.
614 * @channel: channel ID.
615 * @return: 0 on success or negative error code on failure.
617 static int ipu_enable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
619 struct ipu
*ipu
= to_ipu(idmac
);
620 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
624 spin_lock_irqsave(&ipu
->lock
, flags
);
626 /* Reset to buffer 0 */
627 idmac_write_ipureg(ipu
, 1UL << channel
, IPU_CHA_CUR_BUF
);
628 ichan
->active_buffer
= 0;
629 ichan
->status
= IPU_CHANNEL_ENABLED
;
635 ipu_channel_set_priority(ipu
, channel
, true);
640 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
642 idmac_write_icreg(ipu
, reg
| (1UL << channel
), IDMAC_CHA_EN
);
644 ipu_ic_enable_task(ipu
, channel
);
646 spin_unlock_irqrestore(&ipu
->lock
, flags
);
651 * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
652 * @channel: channel ID.
653 * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code.
654 * @width: width of buffer in pixels.
655 * @height: height of buffer in pixels.
656 * @stride: stride length of buffer in pixels.
657 * @rot_mode: rotation mode of buffer. A rotation setting other than
658 * IPU_ROTATE_VERT_FLIP should only be used for input buffers of
660 * @phyaddr_0: buffer 0 physical address.
661 * @phyaddr_1: buffer 1 physical address. Setting this to a value other than
662 * NULL enables double buffering mode.
663 * @return: 0 on success or negative error code on failure.
665 static int ipu_init_channel_buffer(struct idmac_channel
*ichan
,
666 enum pixel_fmt pixel_fmt
,
667 uint16_t width
, uint16_t height
,
669 enum ipu_rotate_mode rot_mode
,
670 dma_addr_t phyaddr_0
, dma_addr_t phyaddr_1
)
672 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
673 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
674 struct ipu
*ipu
= to_ipu(idmac
);
675 union chan_param_mem params
= {};
678 uint32_t stride_bytes
;
680 stride_bytes
= stride
* bytes_per_pixel(pixel_fmt
);
682 if (stride_bytes
% 4) {
684 "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
685 stride
, stride_bytes
);
689 /* IC channel's stride must be a multiple of 8 pixels */
690 if ((channel
<= 13) && (stride
% 8)) {
691 dev_err(ipu
->dev
, "Stride must be 8 pixel multiple\n");
695 /* Build parameter memory data for DMA channel */
696 ipu_ch_param_set_size(¶ms
, pixel_fmt
, width
, height
, stride_bytes
);
697 ipu_ch_param_set_buffer(¶ms
, phyaddr_0
, phyaddr_1
);
698 ipu_ch_param_set_rotation(¶ms
, rot_mode
);
699 /* Some channels (rotation) have restriction on burst length */
701 case IDMAC_IC_7
: /* Hangs with burst 8, 16, other values
702 invalid - Table 44-30 */
704 ipu_ch_param_set_burst_size(¶ms, 8);
709 /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
710 ipu_ch_param_set_burst_size(¶ms
, 16);
717 spin_lock_irqsave(&ipu
->lock
, flags
);
719 ipu_write_param_mem(dma_param_addr(channel
), (uint32_t *)¶ms
, 10);
721 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
724 reg
|= 1UL << channel
;
726 reg
&= ~(1UL << channel
);
728 idmac_write_ipureg(ipu
, reg
, IPU_CHA_DB_MODE_SEL
);
730 ichan
->status
= IPU_CHANNEL_READY
;
732 spin_unlock_irqrestore(&ipu
->lock
, flags
);
738 * ipu_select_buffer() - mark a channel's buffer as ready.
739 * @channel: channel ID.
740 * @buffer_n: buffer number to mark ready.
742 static void ipu_select_buffer(enum ipu_channel channel
, int buffer_n
)
744 /* No locking - this is a write-one-to-set register, cleared by IPU */
746 /* Mark buffer 0 as ready. */
747 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF0_RDY
);
749 /* Mark buffer 1 as ready. */
750 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF1_RDY
);
754 * ipu_update_channel_buffer() - update physical address of a channel buffer.
755 * @channel: channel ID.
756 * @buffer_n: buffer number to update.
757 * 0 or 1 are the only valid values.
758 * @phyaddr: buffer physical address.
759 * @return: Returns 0 on success or negative error code on failure. This
760 * function will fail if the buffer is set to ready.
762 /* Called under spin_lock(_irqsave)(&ichan->lock) */
763 static int ipu_update_channel_buffer(enum ipu_channel channel
,
764 int buffer_n
, dma_addr_t phyaddr
)
769 spin_lock_irqsave(&ipu_data
.lock
, flags
);
772 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
773 if (reg
& (1UL << channel
)) {
774 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
778 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
779 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
780 0x0008UL
, IPU_IMA_ADDR
);
781 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
783 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
784 if (reg
& (1UL << channel
)) {
785 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
789 /* Check if double-buffering is already enabled */
790 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_DB_MODE_SEL
);
792 if (!(reg
& (1UL << channel
)))
793 idmac_write_ipureg(&ipu_data
, reg
| (1UL << channel
),
794 IPU_CHA_DB_MODE_SEL
);
796 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
797 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
798 0x0009UL
, IPU_IMA_ADDR
);
799 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
802 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
807 /* Called under spin_lock_irqsave(&ichan->lock) */
808 static int ipu_submit_channel_buffers(struct idmac_channel
*ichan
,
809 struct idmac_tx_desc
*desc
)
811 struct scatterlist
*sg
;
814 for (i
= 0, sg
= desc
->sg
; i
< 2 && sg
; i
++) {
819 * On first invocation this shouldn't be necessary, the
820 * call to ipu_init_channel_buffer() above will set
821 * addresses for us, so we could make it conditional
822 * on status >= IPU_CHANNEL_ENABLED, but doing it again
823 * shouldn't hurt either.
825 ret
= ipu_update_channel_buffer(ichan
->dma_chan
.chan_id
, i
,
830 ipu_select_buffer(ichan
->dma_chan
.chan_id
, i
);
839 static dma_cookie_t
idmac_tx_submit(struct dma_async_tx_descriptor
*tx
)
841 struct idmac_tx_desc
*desc
= to_tx_desc(tx
);
842 struct idmac_channel
*ichan
= to_idmac_chan(tx
->chan
);
843 struct idmac
*idmac
= to_idmac(tx
->chan
->device
);
844 struct ipu
*ipu
= to_ipu(idmac
);
849 if (!list_empty(&desc
->list
)) {
850 /* The descriptor doesn't belong to client */
851 dev_err(&ichan
->dma_chan
.dev
->device
,
852 "Descriptor %p not prepared!\n", tx
);
856 mutex_lock(&ichan
->chan_mutex
);
858 if (ichan
->status
< IPU_CHANNEL_READY
) {
859 struct idmac_video_param
*video
= &ichan
->params
.video
;
861 * Initial buffer assignment - the first two sg-entries from
862 * the descriptor will end up in the IDMAC buffers
864 dma_addr_t dma_1
= sg_is_last(desc
->sg
) ? 0 :
865 sg_dma_address(&desc
->sg
[1]);
867 WARN_ON(ichan
->sg
[0] || ichan
->sg
[1]);
869 cookie
= ipu_init_channel_buffer(ichan
,
870 video
->out_pixel_fmt
,
875 sg_dma_address(&desc
->sg
[0]),
881 /* ipu->lock can be taken under ichan->lock, but not v.v. */
882 spin_lock_irqsave(&ichan
->lock
, flags
);
884 /* submit_buffers() atomically verifies and fills empty sg slots */
885 cookie
= ipu_submit_channel_buffers(ichan
, desc
);
887 spin_unlock_irqrestore(&ichan
->lock
, flags
);
892 cookie
= ichan
->dma_chan
.cookie
;
897 /* from dmaengine.h: "last cookie value returned to client" */
898 ichan
->dma_chan
.cookie
= cookie
;
900 spin_lock_irqsave(&ichan
->lock
, flags
);
901 list_add_tail(&desc
->list
, &ichan
->queue
);
902 spin_unlock_irqrestore(&ichan
->lock
, flags
);
904 if (ichan
->status
< IPU_CHANNEL_ENABLED
) {
905 int ret
= ipu_enable_channel(idmac
, ichan
);
908 spin_lock_irqsave(&ichan
->lock
, flags
);
909 list_del_init(&desc
->list
);
910 spin_unlock_irqrestore(&ichan
->lock
, flags
);
912 ichan
->dma_chan
.cookie
= cookie
;
919 mutex_unlock(&ichan
->chan_mutex
);
924 /* Called with ichan->chan_mutex held */
925 static int idmac_desc_alloc(struct idmac_channel
*ichan
, int n
)
927 struct idmac_tx_desc
*desc
= vmalloc(n
* sizeof(struct idmac_tx_desc
));
928 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
933 /* No interrupts, just disable the tasklet for a moment */
934 tasklet_disable(&to_ipu(idmac
)->tasklet
);
936 ichan
->n_tx_desc
= n
;
938 INIT_LIST_HEAD(&ichan
->queue
);
939 INIT_LIST_HEAD(&ichan
->free_list
);
942 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
944 memset(txd
, 0, sizeof(*txd
));
945 dma_async_tx_descriptor_init(txd
, &ichan
->dma_chan
);
946 txd
->tx_submit
= idmac_tx_submit
;
947 txd
->chan
= &ichan
->dma_chan
;
948 INIT_LIST_HEAD(&txd
->tx_list
);
950 list_add(&desc
->list
, &ichan
->free_list
);
955 tasklet_enable(&to_ipu(idmac
)->tasklet
);
961 * ipu_init_channel() - initialize an IPU channel.
962 * @idmac: IPU DMAC context.
963 * @ichan: pointer to the channel object.
964 * @return 0 on success or negative error code on failure.
966 static int ipu_init_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
968 union ipu_channel_param
*params
= &ichan
->params
;
970 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
973 struct ipu
*ipu
= to_ipu(idmac
);
974 int ret
= 0, n_desc
= 0;
976 dev_dbg(ipu
->dev
, "init channel = %d\n", channel
);
978 if (channel
!= IDMAC_SDC_0
&& channel
!= IDMAC_SDC_1
&&
979 channel
!= IDMAC_IC_7
)
982 spin_lock_irqsave(&ipu
->lock
, flags
);
987 reg
= idmac_read_icreg(ipu
, IC_CONF
);
988 idmac_write_icreg(ipu
, reg
& ~IC_CONF_CSI_MEM_WR_EN
, IC_CONF
);
992 reg
= idmac_read_ipureg(ipu
, IPU_FS_PROC_FLOW
);
993 idmac_write_ipureg(ipu
, reg
& ~FS_ENC_IN_VALID
, IPU_FS_PROC_FLOW
);
994 ret
= ipu_ic_init_prpenc(ipu
, params
, true);
1003 ipu
->channel_init_mask
|= 1L << channel
;
1005 /* Enable IPU sub module */
1006 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) |
1007 ipu_channel_conf_mask(channel
);
1008 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1010 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1012 if (n_desc
&& !ichan
->desc
)
1013 ret
= idmac_desc_alloc(ichan
, n_desc
);
1015 dump_idmac_reg(ipu
);
1021 * ipu_uninit_channel() - uninitialize an IPU channel.
1022 * @idmac: IPU DMAC context.
1023 * @ichan: pointer to the channel object.
1025 static void ipu_uninit_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
1027 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1028 unsigned long flags
;
1030 unsigned long chan_mask
= 1UL << channel
;
1032 struct ipu
*ipu
= to_ipu(idmac
);
1034 spin_lock_irqsave(&ipu
->lock
, flags
);
1036 if (!(ipu
->channel_init_mask
& chan_mask
)) {
1037 dev_err(ipu
->dev
, "Channel already uninitialized %d\n",
1039 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1043 /* Reset the double buffer */
1044 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
1045 idmac_write_ipureg(ipu
, reg
& ~chan_mask
, IPU_CHA_DB_MODE_SEL
);
1047 ichan
->sec_chan_en
= false;
1051 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1052 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
),
1056 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1057 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_PRPENC_EN
| IC_CONF_PRPENC_CSC1
),
1066 ipu
->channel_init_mask
&= ~(1L << channel
);
1068 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) &
1069 ~ipu_channel_conf_mask(channel
);
1070 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1072 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1074 ichan
->n_tx_desc
= 0;
1080 * ipu_disable_channel() - disable an IPU channel.
1081 * @idmac: IPU DMAC context.
1082 * @ichan: channel object pointer.
1083 * @wait_for_stop: flag to set whether to wait for channel end of frame or
1084 * return immediately.
1085 * @return: 0 on success or negative error code on failure.
1087 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
1090 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1091 struct ipu
*ipu
= to_ipu(idmac
);
1093 unsigned long flags
;
1094 unsigned long chan_mask
= 1UL << channel
;
1095 unsigned int timeout
;
1097 if (wait_for_stop
&& channel
!= IDMAC_SDC_1
&& channel
!= IDMAC_SDC_0
) {
1099 /* This waiting always fails. Related to spurious irq problem */
1100 while ((idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
) & chan_mask
) ||
1101 (ipu_channel_status(ipu
, channel
) == TASK_STAT_ACTIVE
)) {
1107 "Warning: timeout waiting for channel %u to "
1108 "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1109 "busy = 0x%08X, tstat = 0x%08X\n", channel
,
1110 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
1111 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
1112 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
),
1113 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
1117 dev_dbg(ipu
->dev
, "timeout = %d * 10ms\n", 40 - timeout
);
1119 /* SDC BG and FG must be disabled before DMA is disabled */
1120 if (wait_for_stop
&& (channel
== IDMAC_SDC_0
||
1121 channel
== IDMAC_SDC_1
)) {
1123 timeout
&& !ipu_irq_status(ichan
->eof_irq
); timeout
--)
1127 spin_lock_irqsave(&ipu
->lock
, flags
);
1129 /* Disable IC task */
1130 ipu_ic_disable_task(ipu
, channel
);
1132 /* Disable DMA channel(s) */
1133 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
1134 idmac_write_icreg(ipu
, reg
& ~chan_mask
, IDMAC_CHA_EN
);
1137 * Problem (observed with channel DMAIC_7): after enabling the channel
1138 * and initialising buffers, there comes an interrupt with current still
1139 * pointing at buffer 0, whereas it should use buffer 0 first and only
1140 * generate an interrupt when it is done, then current should already
1141 * point to buffer 1. This spurious interrupt also comes on channel
1142 * DMASDC_0. With DMAIC_7 normally, is we just leave the ISR after the
1143 * first interrupt, there comes the second with current correctly
1144 * pointing to buffer 1 this time. But sometimes this second interrupt
1145 * doesn't come and the channel hangs. Clearing BUFx_RDY when disabling
1146 * the channel seems to prevent the channel from hanging, but it doesn't
1147 * prevent the spurious interrupt. This might also be unsafe. Think
1148 * about the IDMAC controller trying to switch to a buffer, when we
1149 * clear the ready bit, and re-enable it a moment later.
1151 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
);
1152 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF0_RDY
);
1153 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF0_RDY
);
1155 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
);
1156 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF1_RDY
);
1157 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF1_RDY
);
1159 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1165 * We have several possibilities here:
1166 * current BUF next BUF
1168 * not last sg next not last sg
1169 * not last sg next last sg
1170 * last sg first sg from next descriptor
1173 * Besides, the descriptor queue might be empty or not. We process all these
1176 static irqreturn_t
idmac_interrupt(int irq
, void *dev_id
)
1178 struct idmac_channel
*ichan
= dev_id
;
1179 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
1180 struct scatterlist
**sg
, *sgnext
, *sgnew
= NULL
;
1181 /* Next transfer descriptor */
1182 struct idmac_tx_desc
*desc
= NULL
, *descnew
;
1183 dma_async_tx_callback callback
;
1184 void *callback_param
;
1186 u32 ready0
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
),
1187 ready1
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
),
1188 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1190 /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1192 pr_debug("IDMAC irq %d\n", irq
);
1193 /* Other interrupts do not interfere with this channel */
1194 spin_lock(&ichan
->lock
);
1196 if (unlikely(chan_id
!= IDMAC_SDC_0
&& chan_id
!= IDMAC_SDC_1
&&
1197 ((curbuf
>> chan_id
) & 1) == ichan
->active_buffer
)) {
1200 /* This doesn't help. See comment in ipu_disable_channel() */
1202 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1203 if (((curbuf
>> chan_id
) & 1) != ichan
->active_buffer
)
1209 spin_unlock(&ichan
->lock
);
1210 dev_dbg(ichan
->dma_chan
.device
->dev
,
1211 "IRQ on active buffer on channel %x, active "
1212 "%d, ready %x, %x, current %x!\n", chan_id
,
1213 ichan
->active_buffer
, ready0
, ready1
, curbuf
);
1218 if (unlikely((ichan
->active_buffer
&& (ready1
>> chan_id
) & 1) ||
1219 (!ichan
->active_buffer
&& (ready0
>> chan_id
) & 1)
1221 spin_unlock(&ichan
->lock
);
1222 dev_dbg(ichan
->dma_chan
.device
->dev
,
1223 "IRQ with active buffer still ready on channel %x, "
1224 "active %d, ready %x, %x!\n", chan_id
,
1225 ichan
->active_buffer
, ready0
, ready1
);
1229 if (unlikely(list_empty(&ichan
->queue
))) {
1230 spin_unlock(&ichan
->lock
);
1231 dev_err(ichan
->dma_chan
.device
->dev
,
1232 "IRQ without queued buffers on channel %x, active %d, "
1233 "ready %x, %x!\n", chan_id
,
1234 ichan
->active_buffer
, ready0
, ready1
);
1239 * active_buffer is a software flag, it shows which buffer we are
1240 * currently expecting back from the hardware, IDMAC should be
1241 * processing the other buffer already
1243 sg
= &ichan
->sg
[ichan
->active_buffer
];
1244 sgnext
= ichan
->sg
[!ichan
->active_buffer
];
1247 * if sgnext == NULL sg must be the last element in a scatterlist and
1248 * queue must be empty
1250 if (unlikely(!sgnext
)) {
1251 if (unlikely(sg_next(*sg
))) {
1252 dev_err(ichan
->dma_chan
.device
->dev
,
1253 "Broken buffer-update locking on channel %x!\n",
1255 /* We'll let the user catch up */
1258 ipu_ic_disable_task(&ipu_data
, chan_id
);
1259 dev_dbg(ichan
->dma_chan
.device
->dev
,
1260 "Underrun on channel %x\n", chan_id
);
1261 ichan
->status
= IPU_CHANNEL_READY
;
1262 /* Continue to check for complete descriptor */
1266 desc
= list_entry(ichan
->queue
.next
, struct idmac_tx_desc
, list
);
1268 /* First calculate and submit the next sg element */
1270 sgnew
= sg_next(sgnext
);
1272 if (unlikely(!sgnew
)) {
1273 /* Start a new scatterlist, if any queued */
1274 if (likely(desc
->list
.next
!= &ichan
->queue
)) {
1275 descnew
= list_entry(desc
->list
.next
,
1276 struct idmac_tx_desc
, list
);
1277 sgnew
= &descnew
->sg
[0];
1281 if (unlikely(!sg_next(*sg
)) || !sgnext
) {
1283 * Last element in scatterlist done, remove from the queue,
1284 * _init for debugging
1286 list_del_init(&desc
->list
);
1292 if (likely(sgnew
)) {
1295 ret
= ipu_update_channel_buffer(chan_id
, ichan
->active_buffer
,
1296 sg_dma_address(*sg
));
1298 dev_err(ichan
->dma_chan
.device
->dev
,
1299 "Failed to update buffer on channel %x buffer %d!\n",
1300 chan_id
, ichan
->active_buffer
);
1302 ipu_select_buffer(chan_id
, ichan
->active_buffer
);
1305 /* Flip the active buffer - even if update above failed */
1306 ichan
->active_buffer
= !ichan
->active_buffer
;
1308 ichan
->completed
= desc
->txd
.cookie
;
1310 callback
= desc
->txd
.callback
;
1311 callback_param
= desc
->txd
.callback_param
;
1313 spin_unlock(&ichan
->lock
);
1315 if (done
&& (desc
->txd
.flags
& DMA_PREP_INTERRUPT
) && callback
)
1316 callback(callback_param
);
1321 static void ipu_gc_tasklet(unsigned long arg
)
1323 struct ipu
*ipu
= (struct ipu
*)arg
;
1326 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1327 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1328 struct idmac_tx_desc
*desc
;
1329 unsigned long flags
;
1332 for (j
= 0; j
< ichan
->n_tx_desc
; j
++) {
1333 desc
= ichan
->desc
+ j
;
1334 spin_lock_irqsave(&ichan
->lock
, flags
);
1335 if (async_tx_test_ack(&desc
->txd
)) {
1336 list_move(&desc
->list
, &ichan
->free_list
);
1337 async_tx_clear_ack(&desc
->txd
);
1339 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1345 * At the time .device_alloc_chan_resources() method is called, we cannot know,
1346 * whether the client will accept the channel. Thus we must only check, if we
1347 * can satisfy client's request but the only real criterion to verify, whether
1348 * the client has accepted our offer is the client_count. That's why we have to
1349 * perform the rest of our allocation tasks on the first call to this function.
1351 static struct dma_async_tx_descriptor
*idmac_prep_slave_sg(struct dma_chan
*chan
,
1352 struct scatterlist
*sgl
, unsigned int sg_len
,
1353 enum dma_data_direction direction
, unsigned long tx_flags
)
1355 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1356 struct idmac_tx_desc
*desc
= NULL
;
1357 struct dma_async_tx_descriptor
*txd
= NULL
;
1358 unsigned long flags
;
1360 /* We only can handle these three channels so far */
1361 if (ichan
->dma_chan
.chan_id
!= IDMAC_SDC_0
&& ichan
->dma_chan
.chan_id
!= IDMAC_SDC_1
&&
1362 ichan
->dma_chan
.chan_id
!= IDMAC_IC_7
)
1365 if (direction
!= DMA_FROM_DEVICE
&& direction
!= DMA_TO_DEVICE
) {
1366 dev_err(chan
->device
->dev
, "Invalid DMA direction %d!\n", direction
);
1370 mutex_lock(&ichan
->chan_mutex
);
1372 spin_lock_irqsave(&ichan
->lock
, flags
);
1373 if (!list_empty(&ichan
->free_list
)) {
1374 desc
= list_entry(ichan
->free_list
.next
,
1375 struct idmac_tx_desc
, list
);
1377 list_del_init(&desc
->list
);
1379 desc
->sg_len
= sg_len
;
1382 txd
->flags
= tx_flags
;
1384 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1386 mutex_unlock(&ichan
->chan_mutex
);
1388 tasklet_schedule(&to_ipu(to_idmac(chan
->device
))->tasklet
);
1393 /* Re-select the current buffer and re-activate the channel */
1394 static void idmac_issue_pending(struct dma_chan
*chan
)
1396 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1397 struct idmac
*idmac
= to_idmac(chan
->device
);
1398 struct ipu
*ipu
= to_ipu(idmac
);
1399 unsigned long flags
;
1401 /* This is not always needed, but doesn't hurt either */
1402 spin_lock_irqsave(&ipu
->lock
, flags
);
1403 ipu_select_buffer(ichan
->dma_chan
.chan_id
, ichan
->active_buffer
);
1404 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1407 * Might need to perform some parts of initialisation from
1408 * ipu_enable_channel(), but not all, we do not want to reset to buffer
1409 * 0, don't need to set priority again either, but re-enabling the task
1410 * and the channel might be a good idea.
1414 static void __idmac_terminate_all(struct dma_chan
*chan
)
1416 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1417 struct idmac
*idmac
= to_idmac(chan
->device
);
1418 unsigned long flags
;
1421 ipu_disable_channel(idmac
, ichan
,
1422 ichan
->status
>= IPU_CHANNEL_ENABLED
);
1424 tasklet_disable(&to_ipu(idmac
)->tasklet
);
1426 /* ichan->queue is modified in ISR, have to spinlock */
1427 spin_lock_irqsave(&ichan
->lock
, flags
);
1428 list_splice_init(&ichan
->queue
, &ichan
->free_list
);
1431 for (i
= 0; i
< ichan
->n_tx_desc
; i
++) {
1432 struct idmac_tx_desc
*desc
= ichan
->desc
+ i
;
1433 if (list_empty(&desc
->list
))
1434 /* Descriptor was prepared, but not submitted */
1435 list_add(&desc
->list
,
1438 async_tx_clear_ack(&desc
->txd
);
1441 ichan
->sg
[0] = NULL
;
1442 ichan
->sg
[1] = NULL
;
1443 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1445 tasklet_enable(&to_ipu(idmac
)->tasklet
);
1447 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1450 static void idmac_terminate_all(struct dma_chan
*chan
)
1452 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1454 mutex_lock(&ichan
->chan_mutex
);
1456 __idmac_terminate_all(chan
);
1458 mutex_unlock(&ichan
->chan_mutex
);
1461 static int idmac_alloc_chan_resources(struct dma_chan
*chan
)
1463 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1464 struct idmac
*idmac
= to_idmac(chan
->device
);
1467 /* dmaengine.c now guarantees to only offer free channels */
1468 BUG_ON(chan
->client_count
> 1);
1469 WARN_ON(ichan
->status
!= IPU_CHANNEL_FREE
);
1472 ichan
->completed
= -ENXIO
;
1474 ret
= ipu_irq_map(ichan
->dma_chan
.chan_id
);
1478 ichan
->eof_irq
= ret
;
1479 ret
= request_irq(ichan
->eof_irq
, idmac_interrupt
, 0,
1480 ichan
->eof_name
, ichan
);
1484 ret
= ipu_init_channel(idmac
, ichan
);
1488 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1490 dev_dbg(&ichan
->dma_chan
.dev
->device
, "Found channel 0x%x, irq %d\n",
1491 ichan
->dma_chan
.chan_id
, ichan
->eof_irq
);
1496 free_irq(ichan
->eof_irq
, ichan
);
1498 ipu_irq_unmap(ichan
->dma_chan
.chan_id
);
1503 static void idmac_free_chan_resources(struct dma_chan
*chan
)
1505 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1506 struct idmac
*idmac
= to_idmac(chan
->device
);
1508 mutex_lock(&ichan
->chan_mutex
);
1510 __idmac_terminate_all(chan
);
1512 if (ichan
->status
> IPU_CHANNEL_FREE
) {
1513 free_irq(ichan
->eof_irq
, ichan
);
1514 ipu_irq_unmap(ichan
->dma_chan
.chan_id
);
1517 ichan
->status
= IPU_CHANNEL_FREE
;
1519 ipu_uninit_channel(idmac
, ichan
);
1521 mutex_unlock(&ichan
->chan_mutex
);
1523 tasklet_schedule(&to_ipu(idmac
)->tasklet
);
1526 static enum dma_status
idmac_is_tx_complete(struct dma_chan
*chan
,
1527 dma_cookie_t cookie
, dma_cookie_t
*done
, dma_cookie_t
*used
)
1529 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1532 *done
= ichan
->completed
;
1534 *used
= chan
->cookie
;
1535 if (cookie
!= chan
->cookie
)
1540 static int __init
ipu_idmac_init(struct ipu
*ipu
)
1542 struct idmac
*idmac
= &ipu
->idmac
;
1543 struct dma_device
*dma
= &idmac
->dma
;
1546 dma_cap_set(DMA_SLAVE
, dma
->cap_mask
);
1547 dma_cap_set(DMA_PRIVATE
, dma
->cap_mask
);
1549 /* Compulsory common fields */
1550 dma
->dev
= ipu
->dev
;
1551 dma
->device_alloc_chan_resources
= idmac_alloc_chan_resources
;
1552 dma
->device_free_chan_resources
= idmac_free_chan_resources
;
1553 dma
->device_is_tx_complete
= idmac_is_tx_complete
;
1554 dma
->device_issue_pending
= idmac_issue_pending
;
1556 /* Compulsory for DMA_SLAVE fields */
1557 dma
->device_prep_slave_sg
= idmac_prep_slave_sg
;
1558 dma
->device_terminate_all
= idmac_terminate_all
;
1560 INIT_LIST_HEAD(&dma
->channels
);
1561 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1562 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1563 struct dma_chan
*dma_chan
= &ichan
->dma_chan
;
1565 spin_lock_init(&ichan
->lock
);
1566 mutex_init(&ichan
->chan_mutex
);
1568 ichan
->status
= IPU_CHANNEL_FREE
;
1569 ichan
->sec_chan_en
= false;
1570 ichan
->completed
= -ENXIO
;
1571 snprintf(ichan
->eof_name
, sizeof(ichan
->eof_name
), "IDMAC EOF %d", i
);
1573 dma_chan
->device
= &idmac
->dma
;
1574 dma_chan
->cookie
= 1;
1575 dma_chan
->chan_id
= i
;
1576 list_add_tail(&ichan
->dma_chan
.device_node
, &dma
->channels
);
1579 idmac_write_icreg(ipu
, 0x00000070, IDMAC_CONF
);
1581 return dma_async_device_register(&idmac
->dma
);
1584 static void ipu_idmac_exit(struct ipu
*ipu
)
1587 struct idmac
*idmac
= &ipu
->idmac
;
1589 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1590 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1592 idmac_terminate_all(&ichan
->dma_chan
);
1593 idmac_prep_slave_sg(&ichan
->dma_chan
, NULL
, 0, DMA_NONE
, 0);
1596 dma_async_device_unregister(&idmac
->dma
);
1599 /*****************************************************************************
1600 * IPU common probe / remove
1603 static int ipu_probe(struct platform_device
*pdev
)
1605 struct ipu_platform_data
*pdata
= pdev
->dev
.platform_data
;
1606 struct resource
*mem_ipu
, *mem_ic
;
1609 spin_lock_init(&ipu_data
.lock
);
1611 mem_ipu
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1612 mem_ic
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1613 if (!pdata
|| !mem_ipu
|| !mem_ic
)
1616 ipu_data
.dev
= &pdev
->dev
;
1618 platform_set_drvdata(pdev
, &ipu_data
);
1620 ret
= platform_get_irq(pdev
, 0);
1624 ipu_data
.irq_fn
= ret
;
1625 ret
= platform_get_irq(pdev
, 1);
1629 ipu_data
.irq_err
= ret
;
1630 ipu_data
.irq_base
= pdata
->irq_base
;
1632 dev_dbg(&pdev
->dev
, "fn irq %u, err irq %u, irq-base %u\n",
1633 ipu_data
.irq_fn
, ipu_data
.irq_err
, ipu_data
.irq_base
);
1635 /* Remap IPU common registers */
1636 ipu_data
.reg_ipu
= ioremap(mem_ipu
->start
,
1637 mem_ipu
->end
- mem_ipu
->start
+ 1);
1638 if (!ipu_data
.reg_ipu
) {
1640 goto err_ioremap_ipu
;
1643 /* Remap Image Converter and Image DMA Controller registers */
1644 ipu_data
.reg_ic
= ioremap(mem_ic
->start
,
1645 mem_ic
->end
- mem_ic
->start
+ 1);
1646 if (!ipu_data
.reg_ic
) {
1648 goto err_ioremap_ic
;
1652 ipu_data
.ipu_clk
= clk_get(&pdev
->dev
, "ipu_clk");
1653 if (IS_ERR(ipu_data
.ipu_clk
)) {
1654 ret
= PTR_ERR(ipu_data
.ipu_clk
);
1658 /* Make sure IPU HSP clock is running */
1659 clk_enable(ipu_data
.ipu_clk
);
1661 /* Disable all interrupts */
1662 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_1
);
1663 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_2
);
1664 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_3
);
1665 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_4
);
1666 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_5
);
1668 dev_dbg(&pdev
->dev
, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev
->name
,
1669 (unsigned long)mem_ipu
->start
, ipu_data
.irq_fn
, ipu_data
.irq_err
);
1671 ret
= ipu_irq_attach_irq(&ipu_data
, pdev
);
1673 goto err_attach_irq
;
1675 /* Initialize DMA engine */
1676 ret
= ipu_idmac_init(&ipu_data
);
1678 goto err_idmac_init
;
1680 tasklet_init(&ipu_data
.tasklet
, ipu_gc_tasklet
, (unsigned long)&ipu_data
);
1682 ipu_data
.dev
= &pdev
->dev
;
1684 dev_dbg(ipu_data
.dev
, "IPU initialized\n");
1690 ipu_irq_detach_irq(&ipu_data
, pdev
);
1691 clk_disable(ipu_data
.ipu_clk
);
1692 clk_put(ipu_data
.ipu_clk
);
1694 iounmap(ipu_data
.reg_ic
);
1696 iounmap(ipu_data
.reg_ipu
);
1699 dev_err(&pdev
->dev
, "Failed to probe IPU: %d\n", ret
);
1703 static int ipu_remove(struct platform_device
*pdev
)
1705 struct ipu
*ipu
= platform_get_drvdata(pdev
);
1707 ipu_idmac_exit(ipu
);
1708 ipu_irq_detach_irq(ipu
, pdev
);
1709 clk_disable(ipu
->ipu_clk
);
1710 clk_put(ipu
->ipu_clk
);
1711 iounmap(ipu
->reg_ic
);
1712 iounmap(ipu
->reg_ipu
);
1713 tasklet_kill(&ipu
->tasklet
);
1714 platform_set_drvdata(pdev
, NULL
);
1720 * We need two MEM resources - with IPU-common and Image Converter registers,
1721 * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1723 static struct platform_driver ipu_platform_driver
= {
1726 .owner
= THIS_MODULE
,
1728 .remove
= ipu_remove
,
1731 static int __init
ipu_init(void)
1733 return platform_driver_probe(&ipu_platform_driver
, ipu_probe
);
1735 subsys_initcall(ipu_init
);
1737 MODULE_DESCRIPTION("IPU core driver");
1738 MODULE_LICENSE("GPL v2");
1739 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1740 MODULE_ALIAS("platform:ipu-core");