3 * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
5 * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/init.h>
13 #include <linux/platform_device.h>
14 #include <linux/err.h>
15 #include <linux/spinlock.h>
16 #include <linux/delay.h>
17 #include <linux/list.h>
18 #include <linux/clk.h>
19 #include <linux/vmalloc.h>
20 #include <linux/string.h>
21 #include <linux/interrupt.h>
26 #include "ipu_intern.h"
28 #define FS_VF_IN_VALID 0x00000002
29 #define FS_ENC_IN_VALID 0x00000001
31 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
35 * There can be only one, we could allocate it dynamically, but then we'd have
36 * to add an extra parameter to some functions, and use something as ugly as
37 * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
40 static struct ipu ipu_data
;
42 #define to_ipu(id) container_of(id, struct ipu, idmac)
44 static u32
__idmac_read_icreg(struct ipu
*ipu
, unsigned long reg
)
46 return __raw_readl(ipu
->reg_ic
+ reg
);
49 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
51 static void __idmac_write_icreg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
53 __raw_writel(value
, ipu
->reg_ic
+ reg
);
56 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
58 static u32
idmac_read_ipureg(struct ipu
*ipu
, unsigned long reg
)
60 return __raw_readl(ipu
->reg_ipu
+ reg
);
63 static void idmac_write_ipureg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
65 __raw_writel(value
, ipu
->reg_ipu
+ reg
);
68 /*****************************************************************************
69 * IPU / IC common functions
71 static void dump_idmac_reg(struct ipu
*ipu
)
73 dev_dbg(ipu
->dev
, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
74 "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
75 idmac_read_icreg(ipu
, IDMAC_CONF
),
76 idmac_read_icreg(ipu
, IC_CONF
),
77 idmac_read_icreg(ipu
, IDMAC_CHA_EN
),
78 idmac_read_icreg(ipu
, IDMAC_CHA_PRI
),
79 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
));
80 dev_dbg(ipu
->dev
, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
81 "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
82 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
83 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
84 idmac_read_ipureg(ipu
, IPU_CHA_CUR_BUF
),
85 idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
),
86 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
89 static uint32_t bytes_per_pixel(enum pixel_fmt fmt
)
92 case IPU_PIX_FMT_GENERIC
: /* generic data */
93 case IPU_PIX_FMT_RGB332
:
94 case IPU_PIX_FMT_YUV420P
:
95 case IPU_PIX_FMT_YUV422P
:
98 case IPU_PIX_FMT_RGB565
:
99 case IPU_PIX_FMT_YUYV
:
100 case IPU_PIX_FMT_UYVY
:
102 case IPU_PIX_FMT_BGR24
:
103 case IPU_PIX_FMT_RGB24
:
105 case IPU_PIX_FMT_GENERIC_32
: /* generic data */
106 case IPU_PIX_FMT_BGR32
:
107 case IPU_PIX_FMT_RGB32
:
108 case IPU_PIX_FMT_ABGR32
:
113 /* Enable direct write to memory by the Camera Sensor Interface */
114 static void ipu_ic_enable_task(struct ipu
*ipu
, enum ipu_channel channel
)
116 uint32_t ic_conf
, mask
;
120 mask
= IC_CONF_PRPENC_EN
;
123 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
128 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) | mask
;
129 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
132 /* Called under spin_lock_irqsave(&ipu_data.lock) */
133 static void ipu_ic_disable_task(struct ipu
*ipu
, enum ipu_channel channel
)
135 uint32_t ic_conf
, mask
;
139 mask
= IC_CONF_PRPENC_EN
;
142 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
147 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) & ~mask
;
148 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
151 static uint32_t ipu_channel_status(struct ipu
*ipu
, enum ipu_channel channel
)
153 uint32_t stat
= TASK_STAT_IDLE
;
154 uint32_t task_stat_reg
= idmac_read_ipureg(ipu
, IPU_TASKS_STAT
);
158 stat
= (task_stat_reg
& TSTAT_CSI2MEM_MASK
) >>
159 TSTAT_CSI2MEM_OFFSET
;
170 struct chan_param_mem_planar
{
208 } __attribute__ ((packed
));
210 struct chan_param_mem_interleaved
{
267 } __attribute__ ((packed
));
269 union chan_param_mem
{
270 struct chan_param_mem_planar pp
;
271 struct chan_param_mem_interleaved ip
;
274 static void ipu_ch_param_set_plane_offset(union chan_param_mem
*params
,
275 u32 u_offset
, u32 v_offset
)
277 params
->pp
.ubo_l
= u_offset
& 0x7ff;
278 params
->pp
.ubo_h
= u_offset
>> 11;
279 params
->pp
.vbo_l
= v_offset
& 0x1ffff;
280 params
->pp
.vbo_h
= v_offset
>> 17;
283 static void ipu_ch_param_set_size(union chan_param_mem
*params
,
284 uint32_t pixel_fmt
, uint16_t width
,
285 uint16_t height
, uint16_t stride
)
290 params
->pp
.fw
= width
- 1;
291 params
->pp
.fh_l
= height
- 1;
292 params
->pp
.fh_h
= (height
- 1) >> 8;
293 params
->pp
.sl
= stride
- 1;
296 case IPU_PIX_FMT_GENERIC
:
297 /*Represents 8-bit Generic data */
301 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
303 case IPU_PIX_FMT_GENERIC_32
:
304 /*Represents 32-bit Generic data */
308 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
310 case IPU_PIX_FMT_RGB565
:
314 params
->ip
.sat
= 2; /* SAT = 32-bit access */
315 params
->ip
.ofs0
= 0; /* Red bit offset */
316 params
->ip
.ofs1
= 5; /* Green bit offset */
317 params
->ip
.ofs2
= 11; /* Blue bit offset */
318 params
->ip
.ofs3
= 16; /* Alpha bit offset */
319 params
->ip
.wid0
= 4; /* Red bit width - 1 */
320 params
->ip
.wid1
= 5; /* Green bit width - 1 */
321 params
->ip
.wid2
= 4; /* Blue bit width - 1 */
323 case IPU_PIX_FMT_BGR24
:
324 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
327 params
->ip
.sat
= 2; /* SAT = 32-bit access */
328 params
->ip
.ofs0
= 0; /* Red bit offset */
329 params
->ip
.ofs1
= 8; /* Green bit offset */
330 params
->ip
.ofs2
= 16; /* Blue bit offset */
331 params
->ip
.ofs3
= 24; /* Alpha bit offset */
332 params
->ip
.wid0
= 7; /* Red bit width - 1 */
333 params
->ip
.wid1
= 7; /* Green bit width - 1 */
334 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
336 case IPU_PIX_FMT_RGB24
:
337 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
340 params
->ip
.sat
= 2; /* SAT = 32-bit access */
341 params
->ip
.ofs0
= 16; /* Red bit offset */
342 params
->ip
.ofs1
= 8; /* Green bit offset */
343 params
->ip
.ofs2
= 0; /* Blue bit offset */
344 params
->ip
.ofs3
= 24; /* Alpha bit offset */
345 params
->ip
.wid0
= 7; /* Red bit width - 1 */
346 params
->ip
.wid1
= 7; /* Green bit width - 1 */
347 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
349 case IPU_PIX_FMT_BGRA32
:
350 case IPU_PIX_FMT_BGR32
:
354 params
->ip
.sat
= 2; /* SAT = 32-bit access */
355 params
->ip
.ofs0
= 8; /* Red bit offset */
356 params
->ip
.ofs1
= 16; /* Green bit offset */
357 params
->ip
.ofs2
= 24; /* Blue bit offset */
358 params
->ip
.ofs3
= 0; /* Alpha bit offset */
359 params
->ip
.wid0
= 7; /* Red bit width - 1 */
360 params
->ip
.wid1
= 7; /* Green bit width - 1 */
361 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
362 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
364 case IPU_PIX_FMT_RGBA32
:
365 case IPU_PIX_FMT_RGB32
:
369 params
->ip
.sat
= 2; /* SAT = 32-bit access */
370 params
->ip
.ofs0
= 24; /* Red bit offset */
371 params
->ip
.ofs1
= 16; /* Green bit offset */
372 params
->ip
.ofs2
= 8; /* Blue bit offset */
373 params
->ip
.ofs3
= 0; /* Alpha bit offset */
374 params
->ip
.wid0
= 7; /* Red bit width - 1 */
375 params
->ip
.wid1
= 7; /* Green bit width - 1 */
376 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
377 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
379 case IPU_PIX_FMT_ABGR32
:
383 params
->ip
.sat
= 2; /* SAT = 32-bit access */
384 params
->ip
.ofs0
= 8; /* Red bit offset */
385 params
->ip
.ofs1
= 16; /* Green bit offset */
386 params
->ip
.ofs2
= 24; /* Blue bit offset */
387 params
->ip
.ofs3
= 0; /* Alpha bit offset */
388 params
->ip
.wid0
= 7; /* Red bit width - 1 */
389 params
->ip
.wid1
= 7; /* Green bit width - 1 */
390 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
391 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
393 case IPU_PIX_FMT_UYVY
:
397 params
->ip
.sat
= 2; /* SAT = 32-bit access */
399 case IPU_PIX_FMT_YUV420P2
:
400 case IPU_PIX_FMT_YUV420P
:
404 params
->ip
.sat
= 2; /* SAT = 32-bit access */
405 u_offset
= stride
* height
;
406 v_offset
= u_offset
+ u_offset
/ 4;
407 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
409 case IPU_PIX_FMT_YVU422P
:
413 params
->ip
.sat
= 2; /* SAT = 32-bit access */
414 v_offset
= stride
* height
;
415 u_offset
= v_offset
+ v_offset
/ 2;
416 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
418 case IPU_PIX_FMT_YUV422P
:
422 params
->ip
.sat
= 2; /* SAT = 32-bit access */
423 u_offset
= stride
* height
;
424 v_offset
= u_offset
+ u_offset
/ 2;
425 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
428 dev_err(ipu_data
.dev
,
429 "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt
);
436 static void ipu_ch_param_set_burst_size(union chan_param_mem
*params
,
437 uint16_t burst_pixels
)
439 params
->pp
.npb
= burst_pixels
- 1;
442 static void ipu_ch_param_set_buffer(union chan_param_mem
*params
,
443 dma_addr_t buf0
, dma_addr_t buf1
)
445 params
->pp
.eba0
= buf0
;
446 params
->pp
.eba1
= buf1
;
449 static void ipu_ch_param_set_rotation(union chan_param_mem
*params
,
450 enum ipu_rotate_mode rotate
)
452 params
->pp
.bam
= rotate
;
455 static void ipu_write_param_mem(uint32_t addr
, uint32_t *data
,
458 for (; num_words
> 0; num_words
--) {
459 dev_dbg(ipu_data
.dev
,
460 "write param mem - addr = 0x%08X, data = 0x%08X\n",
462 idmac_write_ipureg(&ipu_data
, addr
, IPU_IMA_ADDR
);
463 idmac_write_ipureg(&ipu_data
, *data
++, IPU_IMA_DATA
);
465 if ((addr
& 0x7) == 5) {
466 addr
&= ~0x7; /* set to word 0 */
467 addr
+= 8; /* increment to next row */
472 static int calc_resize_coeffs(uint32_t in_size
, uint32_t out_size
,
473 uint32_t *resize_coeff
,
474 uint32_t *downsize_coeff
)
477 uint32_t temp_downsize
;
479 *resize_coeff
= 1 << 13;
480 *downsize_coeff
= 1 << 13;
482 /* Cannot downsize more than 8:1 */
483 if (out_size
<< 3 < in_size
)
486 /* compute downsizing coefficient */
489 while (temp_size
>= out_size
* 2 && temp_downsize
< 2) {
493 *downsize_coeff
= temp_downsize
;
496 * compute resizing coefficient using the following formula:
497 * resize_coeff = M*(SI -1)/(SO - 1)
498 * where M = 2^13, SI - input size, SO - output size
500 *resize_coeff
= (8192L * (temp_size
- 1)) / (out_size
- 1);
501 if (*resize_coeff
>= 16384L) {
502 dev_err(ipu_data
.dev
, "Warning! Overflow on resize coeff.\n");
503 *resize_coeff
= 0x3FFF;
506 dev_dbg(ipu_data
.dev
, "resizing from %u -> %u pixels, "
507 "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size
, out_size
,
508 *downsize_coeff
, *resize_coeff
>= 8192L ? 1 : 0,
509 ((*resize_coeff
& 0x1FFF) * 10000L) / 8192L, *resize_coeff
);
514 static enum ipu_color_space
format_to_colorspace(enum pixel_fmt fmt
)
517 case IPU_PIX_FMT_RGB565
:
518 case IPU_PIX_FMT_BGR24
:
519 case IPU_PIX_FMT_RGB24
:
520 case IPU_PIX_FMT_BGR32
:
521 case IPU_PIX_FMT_RGB32
:
522 return IPU_COLORSPACE_RGB
;
524 return IPU_COLORSPACE_YCBCR
;
528 static int ipu_ic_init_prpenc(struct ipu
*ipu
,
529 union ipu_channel_param
*params
, bool src_is_csi
)
531 uint32_t reg
, ic_conf
;
532 uint32_t downsize_coeff
, resize_coeff
;
533 enum ipu_color_space in_fmt
, out_fmt
;
535 /* Setup vertical resizing */
536 calc_resize_coeffs(params
->video
.in_height
,
537 params
->video
.out_height
,
538 &resize_coeff
, &downsize_coeff
);
539 reg
= (downsize_coeff
<< 30) | (resize_coeff
<< 16);
541 /* Setup horizontal resizing */
542 calc_resize_coeffs(params
->video
.in_width
,
543 params
->video
.out_width
,
544 &resize_coeff
, &downsize_coeff
);
545 reg
|= (downsize_coeff
<< 14) | resize_coeff
;
547 /* Setup color space conversion */
548 in_fmt
= format_to_colorspace(params
->video
.in_pixel_fmt
);
549 out_fmt
= format_to_colorspace(params
->video
.out_pixel_fmt
);
552 * Colourspace conversion unsupported yet - see _init_csc() in
555 if (in_fmt
!= out_fmt
) {
556 dev_err(ipu
->dev
, "Colourspace conversion unsupported!\n");
560 idmac_write_icreg(ipu
, reg
, IC_PRP_ENC_RSC
);
562 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
);
565 ic_conf
&= ~IC_CONF_RWS_EN
;
567 ic_conf
|= IC_CONF_RWS_EN
;
569 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
574 static uint32_t dma_param_addr(uint32_t dma_ch
)
576 /* Channel Parameter Memory */
577 return 0x10000 | (dma_ch
<< 4);
580 static void ipu_channel_set_priority(struct ipu
*ipu
, enum ipu_channel channel
,
583 u32 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_PRI
);
586 reg
|= 1UL << channel
;
588 reg
&= ~(1UL << channel
);
590 idmac_write_icreg(ipu
, reg
, IDMAC_CHA_PRI
);
595 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel
)
602 mask
= IPU_CONF_CSI_EN
| IPU_CONF_IC_EN
;
606 mask
= IPU_CONF_SDC_EN
| IPU_CONF_DI_EN
;
617 * ipu_enable_channel() - enable an IPU channel.
618 * @idmac: IPU DMAC context.
619 * @ichan: IDMAC channel.
620 * @return: 0 on success or negative error code on failure.
622 static int ipu_enable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
624 struct ipu
*ipu
= to_ipu(idmac
);
625 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
629 spin_lock_irqsave(&ipu
->lock
, flags
);
631 /* Reset to buffer 0 */
632 idmac_write_ipureg(ipu
, 1UL << channel
, IPU_CHA_CUR_BUF
);
633 ichan
->active_buffer
= 0;
634 ichan
->status
= IPU_CHANNEL_ENABLED
;
640 ipu_channel_set_priority(ipu
, channel
, true);
645 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
647 idmac_write_icreg(ipu
, reg
| (1UL << channel
), IDMAC_CHA_EN
);
649 ipu_ic_enable_task(ipu
, channel
);
651 spin_unlock_irqrestore(&ipu
->lock
, flags
);
656 * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
657 * @ichan: IDMAC channel.
658 * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code.
659 * @width: width of buffer in pixels.
660 * @height: height of buffer in pixels.
661 * @stride: stride length of buffer in pixels.
662 * @rot_mode: rotation mode of buffer. A rotation setting other than
663 * IPU_ROTATE_VERT_FLIP should only be used for input buffers of
665 * @phyaddr_0: buffer 0 physical address.
666 * @phyaddr_1: buffer 1 physical address. Setting this to a value other than
667 * NULL enables double buffering mode.
668 * @return: 0 on success or negative error code on failure.
670 static int ipu_init_channel_buffer(struct idmac_channel
*ichan
,
671 enum pixel_fmt pixel_fmt
,
672 uint16_t width
, uint16_t height
,
674 enum ipu_rotate_mode rot_mode
,
675 dma_addr_t phyaddr_0
, dma_addr_t phyaddr_1
)
677 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
678 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
679 struct ipu
*ipu
= to_ipu(idmac
);
680 union chan_param_mem params
= {};
683 uint32_t stride_bytes
;
685 stride_bytes
= stride
* bytes_per_pixel(pixel_fmt
);
687 if (stride_bytes
% 4) {
689 "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
690 stride
, stride_bytes
);
694 /* IC channel's stride must be a multiple of 8 pixels */
695 if ((channel
<= IDMAC_IC_13
) && (stride
% 8)) {
696 dev_err(ipu
->dev
, "Stride must be 8 pixel multiple\n");
700 /* Build parameter memory data for DMA channel */
701 ipu_ch_param_set_size(¶ms
, pixel_fmt
, width
, height
, stride_bytes
);
702 ipu_ch_param_set_buffer(¶ms
, phyaddr_0
, phyaddr_1
);
703 ipu_ch_param_set_rotation(¶ms
, rot_mode
);
704 /* Some channels (rotation) have restriction on burst length */
706 case IDMAC_IC_7
: /* Hangs with burst 8, 16, other values
707 invalid - Table 44-30 */
709 ipu_ch_param_set_burst_size(¶ms, 8);
714 /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
715 ipu_ch_param_set_burst_size(¶ms
, 16);
722 spin_lock_irqsave(&ipu
->lock
, flags
);
724 ipu_write_param_mem(dma_param_addr(channel
), (uint32_t *)¶ms
, 10);
726 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
729 reg
|= 1UL << channel
;
731 reg
&= ~(1UL << channel
);
733 idmac_write_ipureg(ipu
, reg
, IPU_CHA_DB_MODE_SEL
);
735 ichan
->status
= IPU_CHANNEL_READY
;
737 spin_unlock_irqrestore(&ipu
->lock
, flags
);
743 * ipu_select_buffer() - mark a channel's buffer as ready.
744 * @channel: channel ID.
745 * @buffer_n: buffer number to mark ready.
747 static void ipu_select_buffer(enum ipu_channel channel
, int buffer_n
)
749 /* No locking - this is a write-one-to-set register, cleared by IPU */
751 /* Mark buffer 0 as ready. */
752 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF0_RDY
);
754 /* Mark buffer 1 as ready. */
755 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF1_RDY
);
759 * ipu_update_channel_buffer() - update physical address of a channel buffer.
760 * @ichan: IDMAC channel.
761 * @buffer_n: buffer number to update.
762 * 0 or 1 are the only valid values.
763 * @phyaddr: buffer physical address.
765 /* Called under spin_lock(_irqsave)(&ichan->lock) */
766 static void ipu_update_channel_buffer(struct idmac_channel
*ichan
,
767 int buffer_n
, dma_addr_t phyaddr
)
769 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
773 spin_lock_irqsave(&ipu_data
.lock
, flags
);
776 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
777 if (reg
& (1UL << channel
)) {
778 ipu_ic_disable_task(&ipu_data
, channel
);
779 ichan
->status
= IPU_CHANNEL_READY
;
782 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
783 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
784 0x0008UL
, IPU_IMA_ADDR
);
785 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
787 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
788 if (reg
& (1UL << channel
)) {
789 ipu_ic_disable_task(&ipu_data
, channel
);
790 ichan
->status
= IPU_CHANNEL_READY
;
793 /* Check if double-buffering is already enabled */
794 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_DB_MODE_SEL
);
796 if (!(reg
& (1UL << channel
)))
797 idmac_write_ipureg(&ipu_data
, reg
| (1UL << channel
),
798 IPU_CHA_DB_MODE_SEL
);
800 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
801 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
802 0x0009UL
, IPU_IMA_ADDR
);
803 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
806 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
809 /* Called under spin_lock_irqsave(&ichan->lock) */
810 static int ipu_submit_buffer(struct idmac_channel
*ichan
,
811 struct idmac_tx_desc
*desc
, struct scatterlist
*sg
, int buf_idx
)
813 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
814 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
816 if (async_tx_test_ack(&desc
->txd
))
820 * On first invocation this shouldn't be necessary, the call to
821 * ipu_init_channel_buffer() above will set addresses for us, so we
822 * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
823 * doing it again shouldn't hurt either.
825 ipu_update_channel_buffer(ichan
, buf_idx
, sg_dma_address(sg
));
827 ipu_select_buffer(chan_id
, buf_idx
);
828 dev_dbg(dev
, "Updated sg %p on channel 0x%x buffer %d\n",
829 sg
, chan_id
, buf_idx
);
834 /* Called under spin_lock_irqsave(&ichan->lock) */
835 static int ipu_submit_channel_buffers(struct idmac_channel
*ichan
,
836 struct idmac_tx_desc
*desc
)
838 struct scatterlist
*sg
;
841 for (i
= 0, sg
= desc
->sg
; i
< 2 && sg
; i
++) {
845 ret
= ipu_submit_buffer(ichan
, desc
, sg
, i
);
856 static dma_cookie_t
idmac_tx_submit(struct dma_async_tx_descriptor
*tx
)
858 struct idmac_tx_desc
*desc
= to_tx_desc(tx
);
859 struct idmac_channel
*ichan
= to_idmac_chan(tx
->chan
);
860 struct idmac
*idmac
= to_idmac(tx
->chan
->device
);
861 struct ipu
*ipu
= to_ipu(idmac
);
862 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
868 if (!list_empty(&desc
->list
)) {
869 /* The descriptor doesn't belong to client */
870 dev_err(dev
, "Descriptor %p not prepared!\n", tx
);
874 mutex_lock(&ichan
->chan_mutex
);
876 async_tx_clear_ack(tx
);
878 if (ichan
->status
< IPU_CHANNEL_READY
) {
879 struct idmac_video_param
*video
= &ichan
->params
.video
;
881 * Initial buffer assignment - the first two sg-entries from
882 * the descriptor will end up in the IDMAC buffers
884 dma_addr_t dma_1
= sg_is_last(desc
->sg
) ? 0 :
885 sg_dma_address(&desc
->sg
[1]);
887 WARN_ON(ichan
->sg
[0] || ichan
->sg
[1]);
889 cookie
= ipu_init_channel_buffer(ichan
,
890 video
->out_pixel_fmt
,
895 sg_dma_address(&desc
->sg
[0]),
901 dev_dbg(dev
, "Submitting sg %p\n", &desc
->sg
[0]);
903 cookie
= ichan
->dma_chan
.cookie
;
908 /* from dmaengine.h: "last cookie value returned to client" */
909 ichan
->dma_chan
.cookie
= cookie
;
912 /* ipu->lock can be taken under ichan->lock, but not v.v. */
913 spin_lock_irqsave(&ichan
->lock
, flags
);
915 list_add_tail(&desc
->list
, &ichan
->queue
);
916 /* submit_buffers() atomically verifies and fills empty sg slots */
917 ret
= ipu_submit_channel_buffers(ichan
, desc
);
919 spin_unlock_irqrestore(&ichan
->lock
, flags
);
926 if (ichan
->status
< IPU_CHANNEL_ENABLED
) {
927 ret
= ipu_enable_channel(idmac
, ichan
);
938 spin_lock_irqsave(&ichan
->lock
, flags
);
939 list_del_init(&desc
->list
);
940 spin_unlock_irqrestore(&ichan
->lock
, flags
);
942 ichan
->dma_chan
.cookie
= cookie
;
946 mutex_unlock(&ichan
->chan_mutex
);
951 /* Called with ichan->chan_mutex held */
952 static int idmac_desc_alloc(struct idmac_channel
*ichan
, int n
)
954 struct idmac_tx_desc
*desc
= vmalloc(n
* sizeof(struct idmac_tx_desc
));
955 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
960 /* No interrupts, just disable the tasklet for a moment */
961 tasklet_disable(&to_ipu(idmac
)->tasklet
);
963 ichan
->n_tx_desc
= n
;
965 INIT_LIST_HEAD(&ichan
->queue
);
966 INIT_LIST_HEAD(&ichan
->free_list
);
969 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
971 memset(txd
, 0, sizeof(*txd
));
972 dma_async_tx_descriptor_init(txd
, &ichan
->dma_chan
);
973 txd
->tx_submit
= idmac_tx_submit
;
975 list_add(&desc
->list
, &ichan
->free_list
);
980 tasklet_enable(&to_ipu(idmac
)->tasklet
);
986 * ipu_init_channel() - initialize an IPU channel.
987 * @idmac: IPU DMAC context.
988 * @ichan: pointer to the channel object.
989 * @return 0 on success or negative error code on failure.
991 static int ipu_init_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
993 union ipu_channel_param
*params
= &ichan
->params
;
995 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
998 struct ipu
*ipu
= to_ipu(idmac
);
999 int ret
= 0, n_desc
= 0;
1001 dev_dbg(ipu
->dev
, "init channel = %d\n", channel
);
1003 if (channel
!= IDMAC_SDC_0
&& channel
!= IDMAC_SDC_1
&&
1004 channel
!= IDMAC_IC_7
)
1007 spin_lock_irqsave(&ipu
->lock
, flags
);
1012 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1013 idmac_write_icreg(ipu
, reg
& ~IC_CONF_CSI_MEM_WR_EN
, IC_CONF
);
1017 reg
= idmac_read_ipureg(ipu
, IPU_FS_PROC_FLOW
);
1018 idmac_write_ipureg(ipu
, reg
& ~FS_ENC_IN_VALID
, IPU_FS_PROC_FLOW
);
1019 ret
= ipu_ic_init_prpenc(ipu
, params
, true);
1028 ipu
->channel_init_mask
|= 1L << channel
;
1030 /* Enable IPU sub module */
1031 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) |
1032 ipu_channel_conf_mask(channel
);
1033 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1035 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1037 if (n_desc
&& !ichan
->desc
)
1038 ret
= idmac_desc_alloc(ichan
, n_desc
);
1040 dump_idmac_reg(ipu
);
1046 * ipu_uninit_channel() - uninitialize an IPU channel.
1047 * @idmac: IPU DMAC context.
1048 * @ichan: pointer to the channel object.
1050 static void ipu_uninit_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
1052 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1053 unsigned long flags
;
1055 unsigned long chan_mask
= 1UL << channel
;
1057 struct ipu
*ipu
= to_ipu(idmac
);
1059 spin_lock_irqsave(&ipu
->lock
, flags
);
1061 if (!(ipu
->channel_init_mask
& chan_mask
)) {
1062 dev_err(ipu
->dev
, "Channel already uninitialized %d\n",
1064 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1068 /* Reset the double buffer */
1069 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
1070 idmac_write_ipureg(ipu
, reg
& ~chan_mask
, IPU_CHA_DB_MODE_SEL
);
1072 ichan
->sec_chan_en
= false;
1076 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1077 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
),
1081 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1082 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_PRPENC_EN
| IC_CONF_PRPENC_CSC1
),
1091 ipu
->channel_init_mask
&= ~(1L << channel
);
1093 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) &
1094 ~ipu_channel_conf_mask(channel
);
1095 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1097 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1099 ichan
->n_tx_desc
= 0;
1105 * ipu_disable_channel() - disable an IPU channel.
1106 * @idmac: IPU DMAC context.
1107 * @ichan: channel object pointer.
1108 * @wait_for_stop: flag to set whether to wait for channel end of frame or
1109 * return immediately.
1110 * @return: 0 on success or negative error code on failure.
1112 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
1115 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1116 struct ipu
*ipu
= to_ipu(idmac
);
1118 unsigned long flags
;
1119 unsigned long chan_mask
= 1UL << channel
;
1120 unsigned int timeout
;
1122 if (wait_for_stop
&& channel
!= IDMAC_SDC_1
&& channel
!= IDMAC_SDC_0
) {
1124 /* This waiting always fails. Related to spurious irq problem */
1125 while ((idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
) & chan_mask
) ||
1126 (ipu_channel_status(ipu
, channel
) == TASK_STAT_ACTIVE
)) {
1132 "Warning: timeout waiting for channel %u to "
1133 "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1134 "busy = 0x%08X, tstat = 0x%08X\n", channel
,
1135 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
1136 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
1137 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
),
1138 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
1142 dev_dbg(ipu
->dev
, "timeout = %d * 10ms\n", 40 - timeout
);
1144 /* SDC BG and FG must be disabled before DMA is disabled */
1145 if (wait_for_stop
&& (channel
== IDMAC_SDC_0
||
1146 channel
== IDMAC_SDC_1
)) {
1148 timeout
&& !ipu_irq_status(ichan
->eof_irq
); timeout
--)
1152 spin_lock_irqsave(&ipu
->lock
, flags
);
1154 /* Disable IC task */
1155 ipu_ic_disable_task(ipu
, channel
);
1157 /* Disable DMA channel(s) */
1158 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
1159 idmac_write_icreg(ipu
, reg
& ~chan_mask
, IDMAC_CHA_EN
);
1162 * Problem (observed with channel DMAIC_7): after enabling the channel
1163 * and initialising buffers, there comes an interrupt with current still
1164 * pointing at buffer 0, whereas it should use buffer 0 first and only
1165 * generate an interrupt when it is done, then current should already
1166 * point to buffer 1. This spurious interrupt also comes on channel
1167 * DMASDC_0. With DMAIC_7 normally, is we just leave the ISR after the
1168 * first interrupt, there comes the second with current correctly
1169 * pointing to buffer 1 this time. But sometimes this second interrupt
1170 * doesn't come and the channel hangs. Clearing BUFx_RDY when disabling
1171 * the channel seems to prevent the channel from hanging, but it doesn't
1172 * prevent the spurious interrupt. This might also be unsafe. Think
1173 * about the IDMAC controller trying to switch to a buffer, when we
1174 * clear the ready bit, and re-enable it a moment later.
1176 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
);
1177 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF0_RDY
);
1178 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF0_RDY
);
1180 reg
= idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
);
1181 idmac_write_ipureg(ipu
, 0, IPU_CHA_BUF1_RDY
);
1182 idmac_write_ipureg(ipu
, reg
& ~(1UL << channel
), IPU_CHA_BUF1_RDY
);
1184 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1189 static struct scatterlist
*idmac_sg_next(struct idmac_channel
*ichan
,
1190 struct idmac_tx_desc
**desc
, struct scatterlist
*sg
)
1192 struct scatterlist
*sgnew
= sg
? sg_next(sg
) : NULL
;
1195 /* next sg-element in this list */
1198 if ((*desc
)->list
.next
== &ichan
->queue
)
1199 /* No more descriptors on the queue */
1202 /* Fetch next descriptor */
1203 *desc
= list_entry((*desc
)->list
.next
, struct idmac_tx_desc
, list
);
1208 * We have several possibilities here:
1209 * current BUF next BUF
1211 * not last sg next not last sg
1212 * not last sg next last sg
1213 * last sg first sg from next descriptor
1216 * Besides, the descriptor queue might be empty or not. We process all these
1219 static irqreturn_t
idmac_interrupt(int irq
, void *dev_id
)
1221 struct idmac_channel
*ichan
= dev_id
;
1222 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
1223 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
1224 struct scatterlist
**sg
, *sgnext
, *sgnew
= NULL
;
1225 /* Next transfer descriptor */
1226 struct idmac_tx_desc
*desc
, *descnew
;
1227 dma_async_tx_callback callback
;
1228 void *callback_param
;
1230 u32 ready0
, ready1
, curbuf
, err
;
1231 unsigned long flags
;
1233 /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1235 dev_dbg(dev
, "IDMAC irq %d, buf %d\n", irq
, ichan
->active_buffer
);
1237 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1239 ready0
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
1240 ready1
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
1241 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1242 err
= idmac_read_ipureg(&ipu_data
, IPU_INT_STAT_4
);
1244 if (err
& (1 << chan_id
)) {
1245 idmac_write_ipureg(&ipu_data
, 1 << chan_id
, IPU_INT_STAT_4
);
1246 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1249 * ichan->sg[0] = ichan->sg[1] = NULL;
1250 * you can force channel re-enable on the next tx_submit(), but
1251 * this is dirty - think about descriptors with multiple
1254 dev_warn(dev
, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
1255 chan_id
, ready0
, ready1
, curbuf
);
1258 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1260 /* Other interrupts do not interfere with this channel */
1261 spin_lock(&ichan
->lock
);
1262 if (unlikely(chan_id
!= IDMAC_SDC_0
&& chan_id
!= IDMAC_SDC_1
&&
1263 ((curbuf
>> chan_id
) & 1) == ichan
->active_buffer
&&
1264 !list_is_last(ichan
->queue
.next
, &ichan
->queue
))) {
1267 /* This doesn't help. See comment in ipu_disable_channel() */
1269 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1270 if (((curbuf
>> chan_id
) & 1) != ichan
->active_buffer
)
1276 spin_unlock(&ichan
->lock
);
1278 "IRQ on active buffer on channel %x, active "
1279 "%d, ready %x, %x, current %x!\n", chan_id
,
1280 ichan
->active_buffer
, ready0
, ready1
, curbuf
);
1284 "Buffer deactivated on channel %x, active "
1285 "%d, ready %x, %x, current %x, rest %d!\n", chan_id
,
1286 ichan
->active_buffer
, ready0
, ready1
, curbuf
, i
);
1289 if (unlikely((ichan
->active_buffer
&& (ready1
>> chan_id
) & 1) ||
1290 (!ichan
->active_buffer
&& (ready0
>> chan_id
) & 1)
1292 spin_unlock(&ichan
->lock
);
1294 "IRQ with active buffer still ready on channel %x, "
1295 "active %d, ready %x, %x!\n", chan_id
,
1296 ichan
->active_buffer
, ready0
, ready1
);
1300 if (unlikely(list_empty(&ichan
->queue
))) {
1301 ichan
->sg
[ichan
->active_buffer
] = NULL
;
1302 spin_unlock(&ichan
->lock
);
1304 "IRQ without queued buffers on channel %x, active %d, "
1305 "ready %x, %x!\n", chan_id
,
1306 ichan
->active_buffer
, ready0
, ready1
);
1311 * active_buffer is a software flag, it shows which buffer we are
1312 * currently expecting back from the hardware, IDMAC should be
1313 * processing the other buffer already
1315 sg
= &ichan
->sg
[ichan
->active_buffer
];
1316 sgnext
= ichan
->sg
[!ichan
->active_buffer
];
1319 spin_unlock(&ichan
->lock
);
1323 desc
= list_entry(ichan
->queue
.next
, struct idmac_tx_desc
, list
);
1326 dev_dbg(dev
, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
1327 irq
, sg_dma_address(*sg
), sgnext
? sg_dma_address(sgnext
) : 0, ichan
->active_buffer
, curbuf
);
1329 /* Find the descriptor of sgnext */
1330 sgnew
= idmac_sg_next(ichan
, &descnew
, *sg
);
1331 if (sgnext
!= sgnew
)
1332 dev_err(dev
, "Submitted buffer %p, next buffer %p\n", sgnext
, sgnew
);
1335 * if sgnext == NULL sg must be the last element in a scatterlist and
1336 * queue must be empty
1338 if (unlikely(!sgnext
)) {
1339 if (!WARN_ON(sg_next(*sg
)))
1340 dev_dbg(dev
, "Underrun on channel %x\n", chan_id
);
1341 ichan
->sg
[!ichan
->active_buffer
] = sgnew
;
1343 if (unlikely(sgnew
)) {
1344 ipu_submit_buffer(ichan
, descnew
, sgnew
, !ichan
->active_buffer
);
1346 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1347 ipu_ic_disable_task(&ipu_data
, chan_id
);
1348 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1349 ichan
->status
= IPU_CHANNEL_READY
;
1350 /* Continue to check for complete descriptor */
1354 /* Calculate and submit the next sg element */
1355 sgnew
= idmac_sg_next(ichan
, &descnew
, sgnew
);
1357 if (unlikely(!sg_next(*sg
)) || !sgnext
) {
1359 * Last element in scatterlist done, remove from the queue,
1360 * _init for debugging
1362 list_del_init(&desc
->list
);
1368 if (likely(sgnew
) &&
1369 ipu_submit_buffer(ichan
, descnew
, sgnew
, ichan
->active_buffer
) < 0) {
1370 callback
= descnew
->txd
.callback
;
1371 callback_param
= descnew
->txd
.callback_param
;
1372 spin_unlock(&ichan
->lock
);
1374 callback(callback_param
);
1375 spin_lock(&ichan
->lock
);
1378 /* Flip the active buffer - even if update above failed */
1379 ichan
->active_buffer
= !ichan
->active_buffer
;
1381 ichan
->completed
= desc
->txd
.cookie
;
1383 callback
= desc
->txd
.callback
;
1384 callback_param
= desc
->txd
.callback_param
;
1386 spin_unlock(&ichan
->lock
);
1388 if (done
&& (desc
->txd
.flags
& DMA_PREP_INTERRUPT
) && callback
)
1389 callback(callback_param
);
1394 static void ipu_gc_tasklet(unsigned long arg
)
1396 struct ipu
*ipu
= (struct ipu
*)arg
;
1399 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1400 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1401 struct idmac_tx_desc
*desc
;
1402 unsigned long flags
;
1403 struct scatterlist
*sg
;
1406 for (j
= 0; j
< ichan
->n_tx_desc
; j
++) {
1407 desc
= ichan
->desc
+ j
;
1408 spin_lock_irqsave(&ichan
->lock
, flags
);
1409 if (async_tx_test_ack(&desc
->txd
)) {
1410 list_move(&desc
->list
, &ichan
->free_list
);
1411 for_each_sg(desc
->sg
, sg
, desc
->sg_len
, k
) {
1412 if (ichan
->sg
[0] == sg
)
1413 ichan
->sg
[0] = NULL
;
1414 else if (ichan
->sg
[1] == sg
)
1415 ichan
->sg
[1] = NULL
;
1417 async_tx_clear_ack(&desc
->txd
);
1419 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1424 /* Allocate and initialise a transfer descriptor. */
1425 static struct dma_async_tx_descriptor
*idmac_prep_slave_sg(struct dma_chan
*chan
,
1426 struct scatterlist
*sgl
, unsigned int sg_len
,
1427 enum dma_data_direction direction
, unsigned long tx_flags
)
1429 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1430 struct idmac_tx_desc
*desc
= NULL
;
1431 struct dma_async_tx_descriptor
*txd
= NULL
;
1432 unsigned long flags
;
1434 /* We only can handle these three channels so far */
1435 if (chan
->chan_id
!= IDMAC_SDC_0
&& chan
->chan_id
!= IDMAC_SDC_1
&&
1436 chan
->chan_id
!= IDMAC_IC_7
)
1439 if (direction
!= DMA_FROM_DEVICE
&& direction
!= DMA_TO_DEVICE
) {
1440 dev_err(chan
->device
->dev
, "Invalid DMA direction %d!\n", direction
);
1444 mutex_lock(&ichan
->chan_mutex
);
1446 spin_lock_irqsave(&ichan
->lock
, flags
);
1447 if (!list_empty(&ichan
->free_list
)) {
1448 desc
= list_entry(ichan
->free_list
.next
,
1449 struct idmac_tx_desc
, list
);
1451 list_del_init(&desc
->list
);
1453 desc
->sg_len
= sg_len
;
1456 txd
->flags
= tx_flags
;
1458 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1460 mutex_unlock(&ichan
->chan_mutex
);
1462 tasklet_schedule(&to_ipu(to_idmac(chan
->device
))->tasklet
);
1467 /* Re-select the current buffer and re-activate the channel */
1468 static void idmac_issue_pending(struct dma_chan
*chan
)
1470 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1471 struct idmac
*idmac
= to_idmac(chan
->device
);
1472 struct ipu
*ipu
= to_ipu(idmac
);
1473 unsigned long flags
;
1475 /* This is not always needed, but doesn't hurt either */
1476 spin_lock_irqsave(&ipu
->lock
, flags
);
1477 ipu_select_buffer(chan
->chan_id
, ichan
->active_buffer
);
1478 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1481 * Might need to perform some parts of initialisation from
1482 * ipu_enable_channel(), but not all, we do not want to reset to buffer
1483 * 0, don't need to set priority again either, but re-enabling the task
1484 * and the channel might be a good idea.
1488 static void __idmac_terminate_all(struct dma_chan
*chan
)
1490 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1491 struct idmac
*idmac
= to_idmac(chan
->device
);
1492 unsigned long flags
;
1495 ipu_disable_channel(idmac
, ichan
,
1496 ichan
->status
>= IPU_CHANNEL_ENABLED
);
1498 tasklet_disable(&to_ipu(idmac
)->tasklet
);
1500 /* ichan->queue is modified in ISR, have to spinlock */
1501 spin_lock_irqsave(&ichan
->lock
, flags
);
1502 list_splice_init(&ichan
->queue
, &ichan
->free_list
);
1505 for (i
= 0; i
< ichan
->n_tx_desc
; i
++) {
1506 struct idmac_tx_desc
*desc
= ichan
->desc
+ i
;
1507 if (list_empty(&desc
->list
))
1508 /* Descriptor was prepared, but not submitted */
1509 list_add(&desc
->list
, &ichan
->free_list
);
1511 async_tx_clear_ack(&desc
->txd
);
1514 ichan
->sg
[0] = NULL
;
1515 ichan
->sg
[1] = NULL
;
1516 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1518 tasklet_enable(&to_ipu(idmac
)->tasklet
);
1520 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1523 static void idmac_terminate_all(struct dma_chan
*chan
)
1525 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1527 mutex_lock(&ichan
->chan_mutex
);
1529 __idmac_terminate_all(chan
);
1531 mutex_unlock(&ichan
->chan_mutex
);
1535 static irqreturn_t
ic_sof_irq(int irq
, void *dev_id
)
1537 struct idmac_channel
*ichan
= dev_id
;
1538 printk(KERN_DEBUG
"Got SOF IRQ %d on Channel %d\n",
1539 irq
, ichan
->dma_chan
.chan_id
);
1540 disable_irq_nosync(irq
);
1544 static irqreturn_t
ic_eof_irq(int irq
, void *dev_id
)
1546 struct idmac_channel
*ichan
= dev_id
;
1547 printk(KERN_DEBUG
"Got EOF IRQ %d on Channel %d\n",
1548 irq
, ichan
->dma_chan
.chan_id
);
1549 disable_irq_nosync(irq
);
1553 static int ic_sof
= -EINVAL
, ic_eof
= -EINVAL
;
1556 static int idmac_alloc_chan_resources(struct dma_chan
*chan
)
1558 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1559 struct idmac
*idmac
= to_idmac(chan
->device
);
1562 /* dmaengine.c now guarantees to only offer free channels */
1563 BUG_ON(chan
->client_count
> 1);
1564 WARN_ON(ichan
->status
!= IPU_CHANNEL_FREE
);
1567 ichan
->completed
= -ENXIO
;
1569 ret
= ipu_irq_map(chan
->chan_id
);
1573 ichan
->eof_irq
= ret
;
1576 * Important to first disable the channel, because maybe someone
1577 * used it before us, e.g., the bootloader
1579 ipu_disable_channel(idmac
, ichan
, true);
1581 ret
= ipu_init_channel(idmac
, ichan
);
1585 ret
= request_irq(ichan
->eof_irq
, idmac_interrupt
, 0,
1586 ichan
->eof_name
, ichan
);
1591 if (chan
->chan_id
== IDMAC_IC_7
) {
1592 ic_sof
= ipu_irq_map(69);
1594 request_irq(ic_sof
, ic_sof_irq
, 0, "IC SOF", ichan
);
1595 ic_eof
= ipu_irq_map(70);
1597 request_irq(ic_eof
, ic_eof_irq
, 0, "IC EOF", ichan
);
1601 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1603 dev_dbg(&chan
->dev
->device
, "Found channel 0x%x, irq %d\n",
1604 chan
->chan_id
, ichan
->eof_irq
);
1609 ipu_uninit_channel(idmac
, ichan
);
1611 ipu_irq_unmap(chan
->chan_id
);
1616 static void idmac_free_chan_resources(struct dma_chan
*chan
)
1618 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1619 struct idmac
*idmac
= to_idmac(chan
->device
);
1621 mutex_lock(&ichan
->chan_mutex
);
1623 __idmac_terminate_all(chan
);
1625 if (ichan
->status
> IPU_CHANNEL_FREE
) {
1627 if (chan
->chan_id
== IDMAC_IC_7
) {
1629 free_irq(ic_sof
, ichan
);
1634 free_irq(ic_eof
, ichan
);
1640 free_irq(ichan
->eof_irq
, ichan
);
1641 ipu_irq_unmap(chan
->chan_id
);
1644 ichan
->status
= IPU_CHANNEL_FREE
;
1646 ipu_uninit_channel(idmac
, ichan
);
1648 mutex_unlock(&ichan
->chan_mutex
);
1650 tasklet_schedule(&to_ipu(idmac
)->tasklet
);
1653 static enum dma_status
idmac_is_tx_complete(struct dma_chan
*chan
,
1654 dma_cookie_t cookie
, dma_cookie_t
*done
, dma_cookie_t
*used
)
1656 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1659 *done
= ichan
->completed
;
1661 *used
= chan
->cookie
;
1662 if (cookie
!= chan
->cookie
)
1667 static int __init
ipu_idmac_init(struct ipu
*ipu
)
1669 struct idmac
*idmac
= &ipu
->idmac
;
1670 struct dma_device
*dma
= &idmac
->dma
;
1673 dma_cap_set(DMA_SLAVE
, dma
->cap_mask
);
1674 dma_cap_set(DMA_PRIVATE
, dma
->cap_mask
);
1676 /* Compulsory common fields */
1677 dma
->dev
= ipu
->dev
;
1678 dma
->device_alloc_chan_resources
= idmac_alloc_chan_resources
;
1679 dma
->device_free_chan_resources
= idmac_free_chan_resources
;
1680 dma
->device_is_tx_complete
= idmac_is_tx_complete
;
1681 dma
->device_issue_pending
= idmac_issue_pending
;
1683 /* Compulsory for DMA_SLAVE fields */
1684 dma
->device_prep_slave_sg
= idmac_prep_slave_sg
;
1685 dma
->device_terminate_all
= idmac_terminate_all
;
1687 INIT_LIST_HEAD(&dma
->channels
);
1688 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1689 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1690 struct dma_chan
*dma_chan
= &ichan
->dma_chan
;
1692 spin_lock_init(&ichan
->lock
);
1693 mutex_init(&ichan
->chan_mutex
);
1695 ichan
->status
= IPU_CHANNEL_FREE
;
1696 ichan
->sec_chan_en
= false;
1697 ichan
->completed
= -ENXIO
;
1698 snprintf(ichan
->eof_name
, sizeof(ichan
->eof_name
), "IDMAC EOF %d", i
);
1700 dma_chan
->device
= &idmac
->dma
;
1701 dma_chan
->cookie
= 1;
1702 dma_chan
->chan_id
= i
;
1703 list_add_tail(&dma_chan
->device_node
, &dma
->channels
);
1706 idmac_write_icreg(ipu
, 0x00000070, IDMAC_CONF
);
1708 return dma_async_device_register(&idmac
->dma
);
1711 static void __exit
ipu_idmac_exit(struct ipu
*ipu
)
1714 struct idmac
*idmac
= &ipu
->idmac
;
1716 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1717 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1719 idmac_terminate_all(&ichan
->dma_chan
);
1720 idmac_prep_slave_sg(&ichan
->dma_chan
, NULL
, 0, DMA_NONE
, 0);
1723 dma_async_device_unregister(&idmac
->dma
);
1726 /*****************************************************************************
1727 * IPU common probe / remove
1730 static int __init
ipu_probe(struct platform_device
*pdev
)
1732 struct ipu_platform_data
*pdata
= pdev
->dev
.platform_data
;
1733 struct resource
*mem_ipu
, *mem_ic
;
1736 spin_lock_init(&ipu_data
.lock
);
1738 mem_ipu
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1739 mem_ic
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1740 if (!pdata
|| !mem_ipu
|| !mem_ic
)
1743 ipu_data
.dev
= &pdev
->dev
;
1745 platform_set_drvdata(pdev
, &ipu_data
);
1747 ret
= platform_get_irq(pdev
, 0);
1751 ipu_data
.irq_fn
= ret
;
1752 ret
= platform_get_irq(pdev
, 1);
1756 ipu_data
.irq_err
= ret
;
1757 ipu_data
.irq_base
= pdata
->irq_base
;
1759 dev_dbg(&pdev
->dev
, "fn irq %u, err irq %u, irq-base %u\n",
1760 ipu_data
.irq_fn
, ipu_data
.irq_err
, ipu_data
.irq_base
);
1762 /* Remap IPU common registers */
1763 ipu_data
.reg_ipu
= ioremap(mem_ipu
->start
,
1764 mem_ipu
->end
- mem_ipu
->start
+ 1);
1765 if (!ipu_data
.reg_ipu
) {
1767 goto err_ioremap_ipu
;
1770 /* Remap Image Converter and Image DMA Controller registers */
1771 ipu_data
.reg_ic
= ioremap(mem_ic
->start
,
1772 mem_ic
->end
- mem_ic
->start
+ 1);
1773 if (!ipu_data
.reg_ic
) {
1775 goto err_ioremap_ic
;
1779 ipu_data
.ipu_clk
= clk_get(&pdev
->dev
, NULL
);
1780 if (IS_ERR(ipu_data
.ipu_clk
)) {
1781 ret
= PTR_ERR(ipu_data
.ipu_clk
);
1785 /* Make sure IPU HSP clock is running */
1786 clk_enable(ipu_data
.ipu_clk
);
1788 /* Disable all interrupts */
1789 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_1
);
1790 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_2
);
1791 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_3
);
1792 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_4
);
1793 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_5
);
1795 dev_dbg(&pdev
->dev
, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev
->name
,
1796 (unsigned long)mem_ipu
->start
, ipu_data
.irq_fn
, ipu_data
.irq_err
);
1798 ret
= ipu_irq_attach_irq(&ipu_data
, pdev
);
1800 goto err_attach_irq
;
1802 /* Initialize DMA engine */
1803 ret
= ipu_idmac_init(&ipu_data
);
1805 goto err_idmac_init
;
1807 tasklet_init(&ipu_data
.tasklet
, ipu_gc_tasklet
, (unsigned long)&ipu_data
);
1809 ipu_data
.dev
= &pdev
->dev
;
1811 dev_dbg(ipu_data
.dev
, "IPU initialized\n");
1817 ipu_irq_detach_irq(&ipu_data
, pdev
);
1818 clk_disable(ipu_data
.ipu_clk
);
1819 clk_put(ipu_data
.ipu_clk
);
1821 iounmap(ipu_data
.reg_ic
);
1823 iounmap(ipu_data
.reg_ipu
);
1826 dev_err(&pdev
->dev
, "Failed to probe IPU: %d\n", ret
);
1830 static int __exit
ipu_remove(struct platform_device
*pdev
)
1832 struct ipu
*ipu
= platform_get_drvdata(pdev
);
1834 ipu_idmac_exit(ipu
);
1835 ipu_irq_detach_irq(ipu
, pdev
);
1836 clk_disable(ipu
->ipu_clk
);
1837 clk_put(ipu
->ipu_clk
);
1838 iounmap(ipu
->reg_ic
);
1839 iounmap(ipu
->reg_ipu
);
1840 tasklet_kill(&ipu
->tasklet
);
1841 platform_set_drvdata(pdev
, NULL
);
1847 * We need two MEM resources - with IPU-common and Image Converter registers,
1848 * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1850 static struct platform_driver ipu_platform_driver
= {
1853 .owner
= THIS_MODULE
,
1855 .remove
= __exit_p(ipu_remove
),
1858 static int __init
ipu_init(void)
1860 return platform_driver_probe(&ipu_platform_driver
, ipu_probe
);
1862 subsys_initcall(ipu_init
);
1864 MODULE_DESCRIPTION("IPU core driver");
1865 MODULE_LICENSE("GPL v2");
1866 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1867 MODULE_ALIAS("platform:ipu-core");