]> git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - drivers/staging/media/ipu3/ipu3-css.c
projects / mirror_ubuntu-kernels.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branches 'for-5.1/upstream-fixes', 'for-5.2/core', 'for-5.2/ish', 'for-5.2...
[mirror_ubuntu-kernels.git] / drivers / staging / media / ipu3 / ipu3-css.c
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Intel Corporation
3
4 #include <linux/device.h>
5 #include <linux/iopoll.h>
6
7 #include "ipu3-css.h"
8 #include "ipu3-css-fw.h"
9 #include "ipu3-css-params.h"
10 #include "ipu3-dmamap.h"
11 #include "ipu3-tables.h"
12
13 /* IRQ configuration */
14 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \
15 IMGU_IRQCTRL_IRQ_SP2 | \
16 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
17 IMGU_IRQCTRL_IRQ_SW_PIN(1))
18
19 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */
20
21 /* Some sane limits for resolutions */
22 #define IPU3_CSS_MIN_RES 32
23 #define IPU3_CSS_MAX_H 3136
24 #define IPU3_CSS_MAX_W 4224
25
26 /* filter size from graph settings is fixed as 4 */
27 #define FILTER_SIZE 4
28 #define MIN_ENVELOPE 8
29
30 /*
31 * pre-allocated buffer size for CSS ABI, auxiliary frames
32 * after BDS and before GDC. Those values should be tuned
33 * to big enough to avoid buffer re-allocation when
34 * streaming to lower streaming latency.
35 */
36 #define CSS_ABI_SIZE 136
37 #define CSS_BDS_SIZE (4480 * 3200 * 3)
38 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8)
39
40 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q))
41 #define IPU3_CSS_FORMAT_FL_IN \
42 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
43 #define IPU3_CSS_FORMAT_FL_OUT \
44 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
45 #define IPU3_CSS_FORMAT_FL_VF \
46 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
47
48 /* Formats supported by IPU3 Camera Sub System */
49 static const struct imgu_css_format imgu_css_formats[] = {
50 {
51 .pixelformat = V4L2_PIX_FMT_NV12,
52 .colorspace = V4L2_COLORSPACE_SRGB,
53 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
54 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
55 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
56 .bytesperpixel_num = 1 * IPU3_CSS_FORMAT_BPP_DEN,
57 .chroma_decim = 4,
58 .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
59 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
60 }, {
61 /* Each 32 bytes contains 25 10-bit pixels */
62 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
63 .colorspace = V4L2_COLORSPACE_RAW,
64 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
65 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
66 .bit_depth = 10,
67 .bytesperpixel_num = 64,
68 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
69 .flags = IPU3_CSS_FORMAT_FL_IN,
70 }, {
71 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
72 .colorspace = V4L2_COLORSPACE_RAW,
73 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
74 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
75 .bit_depth = 10,
76 .bytesperpixel_num = 64,
77 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
78 .flags = IPU3_CSS_FORMAT_FL_IN,
79 }, {
80 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
81 .colorspace = V4L2_COLORSPACE_RAW,
82 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
83 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
84 .bit_depth = 10,
85 .bytesperpixel_num = 64,
86 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
87 .flags = IPU3_CSS_FORMAT_FL_IN,
88 }, {
89 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
90 .colorspace = V4L2_COLORSPACE_RAW,
91 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
92 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
93 .bit_depth = 10,
94 .bytesperpixel_num = 64,
95 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
96 .flags = IPU3_CSS_FORMAT_FL_IN,
97 },
98 };
99
100 static const struct {
101 enum imgu_abi_queue_id qid;
102 size_t ptr_ofs;
103 } imgu_css_queues[IPU3_CSS_QUEUES] = {
104 [IPU3_CSS_QUEUE_IN] = {
105 IMGU_ABI_QUEUE_C_ID,
106 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
107 },
108 [IPU3_CSS_QUEUE_OUT] = {
109 IMGU_ABI_QUEUE_D_ID,
110 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
111 },
112 [IPU3_CSS_QUEUE_VF] = {
113 IMGU_ABI_QUEUE_E_ID,
114 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
115 },
116 [IPU3_CSS_QUEUE_STAT_3A] = {
117 IMGU_ABI_QUEUE_F_ID,
118 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
119 },
120 };
121
122 /* Initialize queue based on given format, adjust format as needed */
123 static int imgu_css_queue_init(struct imgu_css_queue *queue,
124 struct v4l2_pix_format_mplane *fmt, u32 flags)
125 {
126 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
127 unsigned int i;
128 u32 sizeimage;
129
130 INIT_LIST_HEAD(&queue->bufs);
131
132 queue->css_fmt = NULL; /* Disable */
133 if (!fmt)
134 return 0;
135
136 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
137 if (!(imgu_css_formats[i].flags & flags))
138 continue;
139 queue->css_fmt = &imgu_css_formats[i];
140 if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
141 break;
142 }
143 if (!queue->css_fmt)
144 return -EINVAL; /* Could not find any suitable format */
145
146 queue->fmt.mpix = *fmt;
147
148 f->width = ALIGN(clamp_t(u32, f->width,
149 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
150 f->height = ALIGN(clamp_t(u32, f->height,
151 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
152 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
153 if (queue->css_fmt->frame_format != IMGU_ABI_FRAME_FORMAT_RAW_PACKED)
154 f->plane_fmt[0].bytesperline = DIV_ROUND_UP(queue->width_pad *
155 queue->css_fmt->bytesperpixel_num,
156 IPU3_CSS_FORMAT_BPP_DEN);
157 else
158 /* For packed raw, alignment for bpl is by 50 to the width */
159 f->plane_fmt[0].bytesperline =
160 DIV_ROUND_UP(f->width,
161 IPU3_CSS_FORMAT_BPP_DEN) *
162 queue->css_fmt->bytesperpixel_num;
163
164 sizeimage = f->height * f->plane_fmt[0].bytesperline;
165 if (queue->css_fmt->chroma_decim)
166 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
167
168 f->plane_fmt[0].sizeimage = sizeimage;
169 f->field = V4L2_FIELD_NONE;
170 f->num_planes = 1;
171 f->colorspace = queue->css_fmt->colorspace;
172 f->flags = 0;
173 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
174 f->quantization = V4L2_QUANTIZATION_DEFAULT;
175 f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
176 memset(f->reserved, 0, sizeof(f->reserved));
177
178 return 0;
179 }
180
181 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
182 {
183 return q->css_fmt;
184 }
185
186 /******************* css hw *******************/
187
188 /* In the style of writesl() defined in include/asm-generic/io.h */
189 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
190 {
191 if (count >= 4) {
192 const u32 *buf = mem;
193
194 count /= 4;
195 do {
196 writel(*buf++, addr);
197 addr += 4;
198 } while (--count);
199 }
200 }
201
202 /* Wait until register `reg', masked with `mask', becomes `cmp' */
203 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
204 {
205 u32 val;
206
207 return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
208 1000, 100 * 1000);
209 }
210
211 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
212
213 int imgu_css_set_powerup(struct device *dev, void __iomem *base)
214 {
215 static const unsigned int freq = 450;
216 u32 pm_ctrl, state, val;
217
218 dev_dbg(dev, "%s\n", __func__);
219 /* Clear the CSS busy signal */
220 readl(base + IMGU_REG_GP_BUSY);
221 writel(0, base + IMGU_REG_GP_BUSY);
222
223 /* Wait for idle signal */
224 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
225 IMGU_STATE_IDLE_STS)) {
226 dev_err(dev, "failed to set CSS idle\n");
227 goto fail;
228 }
229
230 /* Reset the css */
231 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
232 base + IMGU_REG_PM_CTRL);
233
234 usleep_range(200, 300);
235
236 /** Prepare CSS */
237
238 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
239 state = readl(base + IMGU_REG_STATE);
240
241 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
242 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
243
244 /* Power up CSS using wrapper */
245 if (state & IMGU_STATE_POWER_DOWN) {
246 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
247 base + IMGU_REG_PM_CTRL);
248 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
249 IMGU_PM_CTRL_START, 0)) {
250 dev_err(dev, "failed to power up CSS\n");
251 goto fail;
252 }
253 usleep_range(2000, 3000);
254 } else {
255 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
256 }
257
258 /* Set the busy bit */
259 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
260
261 /* Set CSS clock frequency */
262 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
263 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
264 writel(val, base + IMGU_REG_PM_CTRL);
265 writel(0, base + IMGU_REG_GP_BUSY);
266 if (imgu_hw_wait(base, IMGU_REG_STATE,
267 IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
268 dev_err(dev, "failed to pwrdn CSS\n");
269 goto fail;
270 }
271 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
272 writel(val, base + IMGU_REG_SYSTEM_REQ);
273 writel(1, base + IMGU_REG_GP_BUSY);
274 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
275 base + IMGU_REG_PM_CTRL);
276 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
277 IMGU_STATE_HALT_STS)) {
278 dev_err(dev, "failed to halt CSS\n");
279 goto fail;
280 }
281
282 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
283 base + IMGU_REG_PM_CTRL);
284 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
285 dev_err(dev, "failed to start CSS\n");
286 goto fail;
287 }
288 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
289 base + IMGU_REG_PM_CTRL);
290
291 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */
292 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
293 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
294 writel(val, base + IMGU_REG_PM_CTRL);
295
296 return 0;
297
298 fail:
299 imgu_css_set_powerdown(dev, base);
300 return -EIO;
301 }
302
303 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
304 {
305 dev_dbg(dev, "%s\n", __func__);
306 /* wait for cio idle signal */
307 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
308 IMGU_CIO_GATE_BURST_MASK, 0))
309 dev_warn(dev, "wait cio gate idle timeout");
310
311 /* wait for css idle signal */
312 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
313 IMGU_STATE_IDLE_STS))
314 dev_warn(dev, "wait css idle timeout\n");
315
316 /* do halt-halted handshake with css */
317 writel(1, base + IMGU_REG_GP_HALT);
318 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
319 IMGU_STATE_HALT_STS))
320 dev_warn(dev, "failed to halt css");
321
322 /* de-assert the busy bit */
323 writel(0, base + IMGU_REG_GP_BUSY);
324 }
325
326 static void imgu_css_hw_enable_irq(struct imgu_css *css)
327 {
328 void __iomem *const base = css->base;
329 u32 val, i;
330
331 /* Set up interrupts */
332
333 /*
334 * Enable IRQ on the SP which signals that SP goes to idle
335 * (aka ready state) and set trigger to pulse
336 */
337 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
338 writel(val, base + IMGU_REG_SP_CTRL(0));
339 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
340
341 /* Enable IRQs from the IMGU wrapper */
342 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
343 /* Clear */
344 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
345
346 /* Enable IRQs from main IRQ controller */
347 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
348 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
349 writel(IMGU_IRQCTRL_IRQ_MASK,
350 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
351 writel(IMGU_IRQCTRL_IRQ_MASK,
352 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
353 writel(IMGU_IRQCTRL_IRQ_MASK,
354 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
355 writel(IMGU_IRQCTRL_IRQ_MASK,
356 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
357 /* Wait for write complete */
358 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
359
360 /* Enable IRQs from SP0 and SP1 controllers */
361 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
362 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
363 writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
364 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
365 writel(IMGU_IRQCTRL_IRQ_MASK,
366 base + IMGU_REG_IRQCTRL_ENABLE(i));
367 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
368 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
369 /* Wait for write complete */
370 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
371 }
372 }
373
374 static int imgu_css_hw_init(struct imgu_css *css)
375 {
376 /* For checking that streaming monitor statuses are valid */
377 static const struct {
378 u32 reg;
379 u32 mask;
380 const char *name;
381 } stream_monitors[] = {
382 {
383 IMGU_REG_GP_SP1_STRMON_STAT,
384 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
385 "ISP0 to SP0"
386 }, {
387 IMGU_REG_GP_ISP_STRMON_STAT,
388 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
389 "SP0 to ISP0"
390 }, {
391 IMGU_REG_GP_MOD_STRMON_STAT,
392 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
393 "ISP0 to DMA0"
394 }, {
395 IMGU_REG_GP_ISP_STRMON_STAT,
396 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
397 "DMA0 to ISP0"
398 }, {
399 IMGU_REG_GP_MOD_STRMON_STAT,
400 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
401 "ISP0 to GDC0"
402 }, {
403 IMGU_REG_GP_MOD_STRMON_STAT,
404 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
405 "GDC0 to ISP0"
406 }, {
407 IMGU_REG_GP_MOD_STRMON_STAT,
408 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
409 "SP0 to DMA0"
410 }, {
411 IMGU_REG_GP_SP1_STRMON_STAT,
412 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
413 "DMA0 to SP0"
414 }, {
415 IMGU_REG_GP_MOD_STRMON_STAT,
416 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
417 "SP0 to GDC0"
418 }, {
419 IMGU_REG_GP_MOD_STRMON_STAT,
420 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
421 "GDC0 to SP0"
422 },
423 };
424
425 struct device *dev = css->dev;
426 void __iomem *const base = css->base;
427 u32 val, i;
428
429 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
430 for (i = 0; i < IMGU_NUM_SP; i++) {
431 struct imgu_fw_info *bi =
432 &css->fwp->binary_header[css->fw_sp[i]];
433
434 writel(css->binary[css->fw_sp[i]].daddr,
435 base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
436 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
437 IMGU_CTRL_ICACHE_INV,
438 base + IMGU_REG_SP_CTRL(bi->type));
439 }
440 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
441 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
442 base + IMGU_REG_ISP_CTRL);
443
444 /* Check that IMGU hardware is ready */
445
446 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
447 dev_err(dev, "SP is not idle\n");
448 return -EIO;
449 }
450 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
451 dev_err(dev, "ISP is not idle\n");
452 return -EIO;
453 }
454
455 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
456 val = readl(base + stream_monitors[i].reg);
457 if (val & stream_monitors[i].mask) {
458 dev_err(dev, "error: Stream monitor %s is valid\n",
459 stream_monitors[i].name);
460 return -EIO;
461 }
462 }
463
464 /* Initialize GDC with default values */
465
466 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
467 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
468 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
469 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
470 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
471
472 writel(val0 | (val1 << 16),
473 base + IMGU_REG_GDC_LUT_BASE + i * 8);
474 writel(val2 | (val3 << 16),
475 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
476 }
477
478 return 0;
479 }
480
481 /* Boot the given IPU3 CSS SP */
482 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
483 {
484 void __iomem *const base = css->base;
485 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
486 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
487 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
488 + bi->blob.data_source,
489 .dmem_data_addr = bi->blob.data_target,
490 .dmem_bss_addr = bi->blob.bss_target,
491 .data_size = bi->blob.data_size,
492 .bss_size = bi->blob.bss_size,
493 .sp_id = sp,
494 };
495
496 writes(&dmem_cfg, sizeof(dmem_cfg), base +
497 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
498
499 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
500
501 writel(readl(base + IMGU_REG_SP_CTRL(sp))
502 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
503
504 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
505 + bi->info.sp.sw_state,
506 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
507 return -EIO;
508
509 return 0;
510 }
511
512 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
513 static int imgu_css_hw_start(struct imgu_css *css)
514 {
515 static const u32 event_mask =
516 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
517 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
518 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
519 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
520 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
521 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
522 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
523 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
524 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
525 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
526 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
527 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
528
529 void __iomem *const base = css->base;
530 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
531 unsigned int i;
532
533 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
534
535 /* Start bootloader */
536
537 writel(IMGU_ABI_BL_SWSTATE_BUSY,
538 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
539 writel(IMGU_NUM_SP,
540 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
541
542 for (i = 0; i < IMGU_NUM_SP; i++) {
543 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */
544 struct imgu_fw_info *sp =
545 &css->fwp->binary_header[css->fw_sp[j]];
546 struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
547 .src_addr = css->binary[css->fw_sp[j]].daddr
548 + sp->blob.text_source,
549 .size = sp->blob.text_size,
550 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
551 .dst_addr = IMGU_SP_PMEM_BASE(j),
552 };
553
554 writes(&dma_cmd, sizeof(dma_cmd),
555 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
556 bl->info.bl.dma_cmd_list);
557 }
558
559 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
560
561 writel(readl(base + IMGU_REG_ISP_CTRL)
562 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
563 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
564 + bl->info.bl.sw_state, ~0,
565 IMGU_ABI_BL_SWSTATE_OK)) {
566 dev_err(css->dev, "failed to start bootloader\n");
567 return -EIO;
568 }
569
570 /* Start ISP */
571
572 memset(css->xmem_sp_group_ptrs.vaddr, 0,
573 sizeof(struct imgu_abi_sp_group));
574
575 bi = &css->fwp->binary_header[css->fw_sp[0]];
576
577 writel(css->xmem_sp_group_ptrs.daddr,
578 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
579
580 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
581 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
582 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
583
584 if (imgu_css_hw_start_sp(css, 0))
585 return -EIO;
586
587 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
588 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
589 bi->info.sp.host_sp_queues_initialized);
590 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
591 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
592 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
593 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
594
595 /* Enable all events for all queues */
596
597 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
598 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
599 + bi->info.sp.host_sp_com
600 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
601 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
602 bi->info.sp.host_sp_queues_initialized);
603
604 /* Start SP1 */
605
606 bi = &css->fwp->binary_header[css->fw_sp[1]];
607
608 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
609 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
610
611 if (imgu_css_hw_start_sp(css, 1))
612 return -EIO;
613
614 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
615 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
616
617 return 0;
618 }
619
620 static void imgu_css_hw_stop(struct imgu_css *css)
621 {
622 void __iomem *const base = css->base;
623 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
624
625 /* Stop fw */
626 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
627 base + IMGU_REG_SP_DMEM_BASE(0) +
628 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
629 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
630 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
631 dev_err(css->dev, "wait sp0 idle timeout.\n");
632 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
633 IMGU_ABI_SP_SWSTATE_TERMINATED)
634 dev_err(css->dev, "sp0 is not terminated.\n");
635 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
636 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
637 dev_err(css->dev, "wait isp idle timeout\n");
638 }
639
640 static void imgu_css_hw_cleanup(struct imgu_css *css)
641 {
642 void __iomem *const base = css->base;
643
644 /** Reset CSS **/
645
646 /* Clear the CSS busy signal */
647 readl(base + IMGU_REG_GP_BUSY);
648 writel(0, base + IMGU_REG_GP_BUSY);
649
650 /* Wait for idle signal */
651 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
652 IMGU_STATE_IDLE_STS))
653 dev_err(css->dev, "failed to shut down hw cleanly\n");
654
655 /* Reset the css */
656 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
657 base + IMGU_REG_PM_CTRL);
658
659 usleep_range(200, 300);
660 }
661
662 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
663 {
664 struct imgu_device *imgu = dev_get_drvdata(css->dev);
665 unsigned int i;
666
667 imgu_css_pool_cleanup(imgu,
668 &css->pipes[pipe].pool.parameter_set_info);
669 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.acc);
670 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.gdc);
671 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.obgrid);
672
673 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
674 imgu_css_pool_cleanup(imgu,
675 &css->pipes[pipe].pool.binary_params_p[i]);
676 }
677
678 /*
679 * This function initializes various stages of the
680 * IPU3 CSS ISP pipeline
681 */
682 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
683 {
684 static const int BYPC = 2; /* Bytes per component */
685 static const struct imgu_abi_buffer_sp buffer_sp_init = {
686 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
687 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
688 };
689
690 struct imgu_abi_isp_iterator_config *cfg_iter;
691 struct imgu_abi_isp_ref_config *cfg_ref;
692 struct imgu_abi_isp_dvs_config *cfg_dvs;
693 struct imgu_abi_isp_tnr3_config *cfg_tnr;
694 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
695 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
696
697 const int stage = 0;
698 unsigned int i, j;
699
700 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
701 const struct imgu_fw_info *bi =
702 &css->fwp->binary_header[css_pipe->bindex];
703 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
704
705 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
706 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
707 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
708 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
709
710 struct imgu_abi_isp_stage *isp_stage;
711 struct imgu_abi_sp_stage *sp_stage;
712 struct imgu_abi_sp_group *sp_group;
713
714 const unsigned int bds_width_pad =
715 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
716 2 * IPU3_UAPI_ISP_VEC_ELEMS);
717
718 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
719 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
720 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
721
722 struct imgu_device *imgu = dev_get_drvdata(css->dev);
723
724 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
725
726 /* Configure iterator */
727
728 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
729 &cofs->dmem.iterator,
730 sizeof(*cfg_iter), vaddr);
731 if (!cfg_iter)
732 goto bad_firmware;
733
734 cfg_iter->input_info.res.width =
735 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
736 cfg_iter->input_info.res.height =
737 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
738 cfg_iter->input_info.padded_width =
739 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
740 cfg_iter->input_info.format =
741 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
742 cfg_iter->input_info.raw_bit_depth =
743 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
744 cfg_iter->input_info.raw_bayer_order =
745 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
746 cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
747
748 cfg_iter->internal_info.res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
749 cfg_iter->internal_info.res.height =
750 css_pipe->rect[IPU3_CSS_RECT_BDS].height;
751 cfg_iter->internal_info.padded_width = bds_width_pad;
752 cfg_iter->internal_info.format =
753 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
754 cfg_iter->internal_info.raw_bit_depth =
755 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
756 cfg_iter->internal_info.raw_bayer_order =
757 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
758 cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
759
760 cfg_iter->output_info.res.width =
761 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
762 cfg_iter->output_info.res.height =
763 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
764 cfg_iter->output_info.padded_width =
765 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
766 cfg_iter->output_info.format =
767 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
768 cfg_iter->output_info.raw_bit_depth =
769 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
770 cfg_iter->output_info.raw_bayer_order =
771 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
772 cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
773
774 cfg_iter->vf_info.res.width =
775 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
776 cfg_iter->vf_info.res.height =
777 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
778 cfg_iter->vf_info.padded_width =
779 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
780 cfg_iter->vf_info.format =
781 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
782 cfg_iter->vf_info.raw_bit_depth =
783 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
784 cfg_iter->vf_info.raw_bayer_order =
785 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
786 cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
787
788 cfg_iter->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
789 cfg_iter->dvs_envelope.height =
790 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
791
792 /* Configure reference (delay) frames */
793
794 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
795 &cofs->dmem.ref,
796 sizeof(*cfg_ref), vaddr);
797 if (!cfg_ref)
798 goto bad_firmware;
799
800 cfg_ref->port_b.crop = 0;
801 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
802 cfg_ref->port_b.width =
803 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
804 cfg_ref->port_b.stride =
805 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
806 cfg_ref->width_a_over_b =
807 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
808 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
809 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
810 cfg_ref->ref_frame_addr_y[i] =
811 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
812 cfg_ref->ref_frame_addr_c[i] =
813 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
814 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
815 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
816 }
817 for (; i < IMGU_ABI_FRAMES_REF; i++) {
818 cfg_ref->ref_frame_addr_y[i] = 0;
819 cfg_ref->ref_frame_addr_c[i] = 0;
820 }
821
822 /* Configure DVS (digital video stabilization) */
823
824 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
825 &cofs->dmem.dvs, sizeof(*cfg_dvs),
826 vaddr);
827 if (!cfg_dvs)
828 goto bad_firmware;
829
830 cfg_dvs->num_horizontal_blocks =
831 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
832 IMGU_DVS_BLOCK_W), 2);
833 cfg_dvs->num_vertical_blocks =
834 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
835 IMGU_DVS_BLOCK_H);
836
837 /* Configure TNR (temporal noise reduction) */
838
839 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
840 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
841 &cofs->dmem.tnr3,
842 sizeof(*cfg_tnr),
843 vaddr);
844 if (!cfg_tnr)
845 goto bad_firmware;
846
847 cfg_tnr->port_b.crop = 0;
848 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
849 cfg_tnr->port_b.width =
850 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
851 cfg_tnr->port_b.stride =
852 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
853 cfg_tnr->width_a_over_b =
854 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
855 cfg_tnr->frame_height =
856 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
857 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
858 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
859 cfg_tnr->frame_addr[i] =
860 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
861 .mem[i].daddr;
862 for (; i < IMGU_ABI_FRAMES_TNR; i++)
863 cfg_tnr->frame_addr[i] = 0;
864 }
865
866 /* Configure ref dmem state parameters */
867
868 cfg = IMGU_ABI_PARAM_CLASS_STATE;
869 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
870
871 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
872 &sofs->dmem.ref,
873 sizeof(*cfg_ref_state),
874 vaddr);
875 if (!cfg_ref_state)
876 goto bad_firmware;
877
878 cfg_ref_state->ref_in_buf_idx = 0;
879 cfg_ref_state->ref_out_buf_idx = 1;
880
881 /* Configure tnr dmem state parameters */
882 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
883 cfg_tnr_state =
884 imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
885 &sofs->dmem.tnr3,
886 sizeof(*cfg_tnr_state),
887 vaddr);
888 if (!cfg_tnr_state)
889 goto bad_firmware;
890
891 cfg_tnr_state->in_bufidx = 0;
892 cfg_tnr_state->out_bufidx = 1;
893 cfg_tnr_state->bypass_filter = 0;
894 cfg_tnr_state->total_frame_counter = 0;
895 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
896 cfg_tnr_state->buffer_frame_counter[i] = 0;
897 }
898
899 /* Configure ISP stage */
900
901 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
902 memset(isp_stage, 0, sizeof(*isp_stage));
903 isp_stage->blob_info = bi->blob;
904 isp_stage->binary_info = bi->info.isp.sp;
905 strscpy(isp_stage->binary_name,
906 (char *)css->fwp + bi->blob.prog_name_offset,
907 sizeof(isp_stage->binary_name));
908 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
909 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
910 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
911 isp_stage->mem_initializers.params[i][j].address =
912 css_pipe->binary_params_cs[i - 1][j].daddr;
913
914 /* Configure SP stage */
915
916 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
917 memset(sp_stage, 0, sizeof(*sp_stage));
918
919 sp_stage->frames.in.buf_attr = buffer_sp_init;
920 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
921 sp_stage->frames.out[i].buf_attr = buffer_sp_init;
922 sp_stage->frames.out_vf.buf_attr = buffer_sp_init;
923 sp_stage->frames.s3a_buf = buffer_sp_init;
924 sp_stage->frames.dvs_buf = buffer_sp_init;
925
926 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
927 sp_stage->num = stage;
928 sp_stage->isp_online = 0;
929 sp_stage->isp_copy_vf = 0;
930 sp_stage->isp_copy_output = 0;
931
932 sp_stage->enable.vf_output = css_pipe->vf_output_en;
933
934 sp_stage->frames.effective_in_res.width =
935 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
936 sp_stage->frames.effective_in_res.height =
937 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
938 sp_stage->frames.in.info.res.width =
939 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
940 sp_stage->frames.in.info.res.height =
941 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
942 sp_stage->frames.in.info.padded_width =
943 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
944 sp_stage->frames.in.info.format =
945 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
946 sp_stage->frames.in.info.raw_bit_depth =
947 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
948 sp_stage->frames.in.info.raw_bayer_order =
949 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
950 sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
951 sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
952 sp_stage->frames.in.buf_attr.buf_type =
953 IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
954
955 sp_stage->frames.out[0].info.res.width =
956 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
957 sp_stage->frames.out[0].info.res.height =
958 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
959 sp_stage->frames.out[0].info.padded_width =
960 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
961 sp_stage->frames.out[0].info.format =
962 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
963 sp_stage->frames.out[0].info.raw_bit_depth =
964 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
965 sp_stage->frames.out[0].info.raw_bayer_order =
966 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
967 sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
968 sp_stage->frames.out[0].planes.nv.uv.offset =
969 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad *
970 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
971 sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
972 sp_stage->frames.out[0].buf_attr.buf_type =
973 IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
974
975 sp_stage->frames.out[1].buf_attr.buf_src.queue_id =
976 IMGU_ABI_QUEUE_EVENT_ID;
977
978 sp_stage->frames.internal_frame_info.res.width =
979 css_pipe->rect[IPU3_CSS_RECT_BDS].width;
980 sp_stage->frames.internal_frame_info.res.height =
981 css_pipe->rect[IPU3_CSS_RECT_BDS].height;
982 sp_stage->frames.internal_frame_info.padded_width = bds_width_pad;
983
984 sp_stage->frames.internal_frame_info.format =
985 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
986 sp_stage->frames.internal_frame_info.raw_bit_depth =
987 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
988 sp_stage->frames.internal_frame_info.raw_bayer_order =
989 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
990 sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
991
992 sp_stage->frames.out_vf.info.res.width =
993 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
994 sp_stage->frames.out_vf.info.res.height =
995 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
996 sp_stage->frames.out_vf.info.padded_width =
997 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
998 sp_stage->frames.out_vf.info.format =
999 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1000 sp_stage->frames.out_vf.info.raw_bit_depth =
1001 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
1002 sp_stage->frames.out_vf.info.raw_bayer_order =
1003 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
1004 sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
1005 sp_stage->frames.out_vf.planes.yuv.u.offset =
1006 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
1007 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
1008 sp_stage->frames.out_vf.planes.yuv.v.offset =
1009 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
1010 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4;
1011 sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
1012 sp_stage->frames.out_vf.buf_attr.buf_type =
1013 IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
1014
1015 sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
1016 sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
1017
1018 sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
1019 sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
1020
1021 sp_stage->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
1022 sp_stage->dvs_envelope.height =
1023 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
1024
1025 sp_stage->isp_pipe_version =
1026 bi->info.isp.sp.pipeline.isp_pipe_version;
1027 sp_stage->isp_deci_log_factor =
1028 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
1029 IMGU_MAX_BQ_GRID_WIDTH),
1030 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
1031 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
1032 sp_stage->isp_vf_downscale_bits = 0;
1033 sp_stage->if_config_index = 255;
1034 sp_stage->sp_enable_xnr = 0;
1035 sp_stage->num_stripes = stripes;
1036 sp_stage->enable.s3a = 1;
1037 sp_stage->enable.dvs_stats = 0;
1038
1039 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
1040 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
1041 sp_stage->isp_stage_addr =
1042 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
1043
1044 /* Configure SP group */
1045
1046 sp_group = css->xmem_sp_group_ptrs.vaddr;
1047 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1048
1049 sp_group->pipe[pipe].num_stages = 1;
1050 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1051 sp_group->pipe[pipe].thread_id = pipe;
1052 sp_group->pipe[pipe].pipe_num = pipe;
1053 sp_group->pipe[pipe].num_execs = -1;
1054 sp_group->pipe[pipe].pipe_qos_config = -1;
1055 sp_group->pipe[pipe].required_bds_factor = 0;
1056 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1057 sp_group->pipe[pipe].inout_port_config =
1058 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1059 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1060 sp_group->pipe[pipe].scaler_pp_lut = 0;
1061 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1062 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1063 sp_group->pipe[pipe].sp_stage_addr[stage] =
1064 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1065 sp_group->pipe[pipe].pipe_config =
1066 bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1067 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1068
1069 /* Initialize parameter pools */
1070
1071 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1072 sizeof(struct imgu_abi_parameter_set_info)) ||
1073 imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1074 sizeof(struct imgu_abi_acc_param)) ||
1075 imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1076 sizeof(struct imgu_abi_gdc_warp_param) *
1077 3 * cfg_dvs->num_horizontal_blocks / 2 *
1078 cfg_dvs->num_vertical_blocks) ||
1079 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1080 imgu_css_fw_obgrid_size(
1081 &css->fwp->binary_header[css_pipe->bindex])))
1082 goto out_of_memory;
1083
1084 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1085 if (imgu_css_pool_init(imgu,
1086 &css_pipe->pool.binary_params_p[i],
1087 bi->info.isp.sp.mem_initializers.params
1088 [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1089 goto out_of_memory;
1090
1091 return 0;
1092
1093 bad_firmware:
1094 imgu_css_pipeline_cleanup(css, pipe);
1095 return -EPROTO;
1096
1097 out_of_memory:
1098 imgu_css_pipeline_cleanup(css, pipe);
1099 return -ENOMEM;
1100 }
1101
1102 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1103 {
1104 static const unsigned int sp;
1105 void __iomem *const base = css->base;
1106 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1107 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1108 bi->info.sp.host_sp_queue;
1109
1110 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1111 readb(&q->host2sp_evtq_info.end);
1112 }
1113
1114 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1115 static int imgu_css_queue_data(struct imgu_css *css,
1116 int queue, int thread, u32 data)
1117 {
1118 static const unsigned int sp;
1119 void __iomem *const base = css->base;
1120 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1121 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1122 bi->info.sp.host_sp_queue;
1123 u8 size, start, end, end2;
1124
1125 if (queue >= 0) {
1126 size = readb(&q->host2sp_bufq_info[thread][queue].size);
1127 start = readb(&q->host2sp_bufq_info[thread][queue].start);
1128 end = readb(&q->host2sp_bufq_info[thread][queue].end);
1129 } else {
1130 size = readb(&q->host2sp_evtq_info.size);
1131 start = readb(&q->host2sp_evtq_info.start);
1132 end = readb(&q->host2sp_evtq_info.end);
1133 }
1134
1135 if (size == 0)
1136 return -EIO;
1137
1138 end2 = (end + 1) % size;
1139 if (end2 == start)
1140 return -EBUSY; /* Queue full */
1141
1142 if (queue >= 0) {
1143 writel(data, &q->host2sp_bufq[thread][queue][end]);
1144 writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1145 } else {
1146 writel(data, &q->host2sp_evtq[end]);
1147 writeb(end2, &q->host2sp_evtq_info.end);
1148 }
1149
1150 return 0;
1151 }
1152
1153 /* Receive data using given buffer queue, or if queue < 0, event queue. */
1154 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1155 {
1156 static const unsigned int sp;
1157 void __iomem *const base = css->base;
1158 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1159 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1160 bi->info.sp.host_sp_queue;
1161 u8 size, start, end, start2;
1162
1163 if (queue >= 0) {
1164 size = readb(&q->sp2host_bufq_info[queue].size);
1165 start = readb(&q->sp2host_bufq_info[queue].start);
1166 end = readb(&q->sp2host_bufq_info[queue].end);
1167 } else {
1168 size = readb(&q->sp2host_evtq_info.size);
1169 start = readb(&q->sp2host_evtq_info.start);
1170 end = readb(&q->sp2host_evtq_info.end);
1171 }
1172
1173 if (size == 0)
1174 return -EIO;
1175
1176 if (end == start)
1177 return -EBUSY; /* Queue empty */
1178
1179 start2 = (start + 1) % size;
1180
1181 if (queue >= 0) {
1182 *data = readl(&q->sp2host_bufq[queue][start]);
1183 writeb(start2, &q->sp2host_bufq_info[queue].start);
1184 } else {
1185 int r;
1186
1187 *data = readl(&q->sp2host_evtq[start]);
1188 writeb(start2, &q->sp2host_evtq_info.start);
1189
1190 /* Acknowledge events dequeued from event queue */
1191 r = imgu_css_queue_data(css, queue, 0,
1192 IMGU_ABI_EVENT_EVENT_DEQUEUED);
1193 if (r < 0)
1194 return r;
1195 }
1196
1197 return 0;
1198 }
1199
1200 /* Free binary-specific resources */
1201 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1202 {
1203 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1204 unsigned int i, j;
1205
1206 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1207
1208 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1209 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1210 imgu_dmamap_free(imgu,
1211 &css_pipe->binary_params_cs[j][i]);
1212
1213 j = IPU3_CSS_AUX_FRAME_REF;
1214 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1215 imgu_dmamap_free(imgu,
1216 &css_pipe->aux_frames[j].mem[i]);
1217
1218 j = IPU3_CSS_AUX_FRAME_TNR;
1219 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1220 imgu_dmamap_free(imgu,
1221 &css_pipe->aux_frames[j].mem[i]);
1222 }
1223
1224 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1225 {
1226 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1227 unsigned int i, j;
1228
1229 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1230
1231 for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1232 j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1233 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1234 if (!imgu_dmamap_alloc(imgu,
1235 &css_pipe->binary_params_cs[j - 1][i],
1236 CSS_ABI_SIZE))
1237 goto out_of_memory;
1238
1239 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1240 if (!imgu_dmamap_alloc(imgu,
1241 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].
1242 mem[i], CSS_BDS_SIZE))
1243 goto out_of_memory;
1244
1245 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1246 if (!imgu_dmamap_alloc(imgu,
1247 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].
1248 mem[i], CSS_GDC_SIZE))
1249 goto out_of_memory;
1250
1251 return 0;
1252
1253 out_of_memory:
1254 imgu_css_binary_cleanup(css, pipe);
1255 return -ENOMEM;
1256 }
1257
1258 /* allocate binary-specific resources */
1259 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1260 {
1261 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1262 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1263 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1264 int i, j, size;
1265 static const int BYPC = 2; /* Bytes per component */
1266 unsigned int w, h;
1267
1268 /* Allocate parameter memory blocks for this binary */
1269
1270 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1271 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1272 if (imgu_css_dma_buffer_resize(
1273 imgu,
1274 &css_pipe->binary_params_cs[j - 1][i],
1275 bi->info.isp.sp.mem_initializers.params[j][i].size))
1276 goto out_of_memory;
1277 }
1278
1279 /* Allocate internal frame buffers */
1280
1281 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1282 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1283 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1284 css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1285 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1286 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1287 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1288 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1289 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1290 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1291 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1292 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1293 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1294 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1295 if (imgu_css_dma_buffer_resize(
1296 imgu,
1297 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1298 size))
1299 goto out_of_memory;
1300
1301 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1302 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1303 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1304 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1305 bi->info.isp.sp.block.block_width *
1306 IPU3_UAPI_ISP_VEC_ELEMS);
1307 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1308 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1309 bi->info.isp.sp.block.output_block_height);
1310
1311 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1312 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1313 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1314 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */
1315 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1316 if (imgu_css_dma_buffer_resize(
1317 imgu,
1318 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1319 size))
1320 goto out_of_memory;
1321
1322 return 0;
1323
1324 out_of_memory:
1325 imgu_css_binary_cleanup(css, pipe);
1326 return -ENOMEM;
1327 }
1328
1329 int imgu_css_start_streaming(struct imgu_css *css)
1330 {
1331 u32 data;
1332 int r, pipe;
1333
1334 if (css->streaming)
1335 return -EPROTO;
1336
1337 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1338 r = imgu_css_binary_setup(css, pipe);
1339 if (r < 0)
1340 return r;
1341 }
1342
1343 r = imgu_css_hw_init(css);
1344 if (r < 0)
1345 return r;
1346
1347 r = imgu_css_hw_start(css);
1348 if (r < 0)
1349 goto fail;
1350
1351 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1352 r = imgu_css_pipeline_init(css, pipe);
1353 if (r < 0)
1354 goto fail;
1355 }
1356
1357 css->streaming = true;
1358
1359 imgu_css_hw_enable_irq(css);
1360
1361 /* Initialize parameters to default */
1362 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1363 r = imgu_css_set_parameters(css, pipe, NULL);
1364 if (r < 0)
1365 goto fail;
1366 }
1367
1368 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1369 ;
1370 if (r != -EBUSY)
1371 goto fail;
1372
1373 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1374 ;
1375 if (r != -EBUSY)
1376 goto fail;
1377
1378 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1379 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1380 IMGU_ABI_EVENT_START_STREAM |
1381 pipe << 16);
1382 if (r < 0)
1383 goto fail;
1384 }
1385
1386 return 0;
1387
1388 fail:
1389 css->streaming = false;
1390 imgu_css_hw_cleanup(css);
1391 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1392 imgu_css_pipeline_cleanup(css, pipe);
1393 imgu_css_binary_cleanup(css, pipe);
1394 }
1395
1396 return r;
1397 }
1398
1399 void imgu_css_stop_streaming(struct imgu_css *css)
1400 {
1401 struct imgu_css_buffer *b, *b0;
1402 int q, r, pipe;
1403
1404 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1405 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1406 IMGU_ABI_EVENT_STOP_STREAM);
1407 if (r < 0)
1408 dev_warn(css->dev, "failed on stop stream event\n");
1409 }
1410
1411 if (!css->streaming)
1412 return;
1413
1414 imgu_css_hw_stop(css);
1415
1416 imgu_css_hw_cleanup(css);
1417
1418 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1419 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1420
1421 imgu_css_pipeline_cleanup(css, pipe);
1422
1423 spin_lock(&css_pipe->qlock);
1424 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1425 list_for_each_entry_safe(b, b0,
1426 &css_pipe->queue[q].bufs,
1427 list) {
1428 b->state = IPU3_CSS_BUFFER_FAILED;
1429 list_del(&b->list);
1430 }
1431 spin_unlock(&css_pipe->qlock);
1432 }
1433
1434 css->streaming = false;
1435 }
1436
1437 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1438 {
1439 int q;
1440 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1441
1442 spin_lock(&css_pipe->qlock);
1443 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1444 if (!list_empty(&css_pipe->queue[q].bufs))
1445 break;
1446 spin_unlock(&css_pipe->qlock);
1447 return (q == IPU3_CSS_QUEUES);
1448 }
1449
1450 bool imgu_css_queue_empty(struct imgu_css *css)
1451 {
1452 unsigned int pipe;
1453 bool ret = 0;
1454
1455 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1456 ret &= imgu_css_pipe_queue_empty(css, pipe);
1457
1458 return ret;
1459 }
1460
1461 bool imgu_css_is_streaming(struct imgu_css *css)
1462 {
1463 return css->streaming;
1464 }
1465
1466 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1467 {
1468 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1469 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1470 unsigned int p, q, i;
1471
1472 /* Allocate and map common structures with imgu hardware */
1473 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1474 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1475 if (!imgu_dmamap_alloc(imgu,
1476 &css_pipe->
1477 xmem_sp_stage_ptrs[p][i],
1478 sizeof(struct imgu_abi_sp_stage)))
1479 return -ENOMEM;
1480 if (!imgu_dmamap_alloc(imgu,
1481 &css_pipe->
1482 xmem_isp_stage_ptrs[p][i],
1483 sizeof(struct imgu_abi_isp_stage)))
1484 return -ENOMEM;
1485 }
1486
1487 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1488 ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1489 IMGU_ABI_ISP_DDR_WORD_BYTES)))
1490 return -ENOMEM;
1491
1492 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1493 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1494
1495 for (i = 0; i < abi_buf_num; i++)
1496 if (!imgu_dmamap_alloc(imgu,
1497 &css_pipe->abi_buffers[q][i],
1498 sizeof(struct imgu_abi_buffer)))
1499 return -ENOMEM;
1500 }
1501
1502 if (imgu_css_binary_preallocate(css, pipe)) {
1503 imgu_css_binary_cleanup(css, pipe);
1504 return -ENOMEM;
1505 }
1506
1507 return 0;
1508 }
1509
1510 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1511 {
1512 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1513 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1514 unsigned int p, q, i, abi_buf_num;
1515
1516 imgu_css_binary_cleanup(css, pipe);
1517
1518 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1519 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1520 for (i = 0; i < abi_buf_num; i++)
1521 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1522 }
1523
1524 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1525 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1526 imgu_dmamap_free(imgu,
1527 &css_pipe->xmem_sp_stage_ptrs[p][i]);
1528 imgu_dmamap_free(imgu,
1529 &css_pipe->xmem_isp_stage_ptrs[p][i]);
1530 }
1531
1532 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1533 }
1534
1535 void imgu_css_cleanup(struct imgu_css *css)
1536 {
1537 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1538 unsigned int pipe;
1539
1540 imgu_css_stop_streaming(css);
1541 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1542 imgu_css_pipe_cleanup(css, pipe);
1543 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1544 imgu_css_fw_cleanup(css);
1545 }
1546
1547 int imgu_css_init(struct device *dev, struct imgu_css *css,
1548 void __iomem *base, int length)
1549 {
1550 struct imgu_device *imgu = dev_get_drvdata(dev);
1551 int r, q, pipe;
1552
1553 /* Initialize main data structure */
1554 css->dev = dev;
1555 css->base = base;
1556 css->iomem_length = length;
1557
1558 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1559 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1560
1561 css_pipe->vf_output_en = false;
1562 spin_lock_init(&css_pipe->qlock);
1563 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1564 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1565 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1566 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1567 if (r)
1568 return r;
1569 }
1570 r = imgu_css_map_init(css, pipe);
1571 if (r) {
1572 imgu_css_cleanup(css);
1573 return r;
1574 }
1575 }
1576 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1577 sizeof(struct imgu_abi_sp_group)))
1578 return -ENOMEM;
1579
1580 r = imgu_css_fw_init(css);
1581 if (r)
1582 return r;
1583
1584 return 0;
1585 }
1586
1587 static u32 imgu_css_adjust(u32 res, u32 align)
1588 {
1589 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1590
1591 return DIV_ROUND_CLOSEST(val, align) * align;
1592 }
1593
1594 /* Select a binary matching the required resolutions and formats */
1595 static int imgu_css_find_binary(struct imgu_css *css,
1596 unsigned int pipe,
1597 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1598 struct v4l2_rect rects[IPU3_CSS_RECTS])
1599 {
1600 const int binary_nr = css->fwp->file_header.binary_nr;
1601 unsigned int binary_mode =
1602 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1603 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1604 const struct v4l2_pix_format_mplane *in =
1605 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1606 const struct v4l2_pix_format_mplane *out =
1607 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1608 const struct v4l2_pix_format_mplane *vf =
1609 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1610 u32 stripe_w = 0, stripe_h = 0;
1611 const char *name;
1612 int i, j;
1613
1614 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1615 return -EINVAL;
1616
1617 /* Find out the strip size boundary */
1618 for (i = 0; i < binary_nr; i++) {
1619 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1620
1621 u32 max_width = bi->info.isp.sp.output.max_width;
1622 u32 max_height = bi->info.isp.sp.output.max_height;
1623
1624 if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1625 stripe_w = stripe_w ?
1626 min(stripe_w, max_width) : max_width;
1627 stripe_h = stripe_h ?
1628 min(stripe_h, max_height) : max_height;
1629 }
1630 }
1631
1632 for (i = 0; i < binary_nr; i++) {
1633 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1634 enum imgu_abi_frame_format q_fmt;
1635
1636 name = (void *)css->fwp + bi->blob.prog_name_offset;
1637
1638 /* Check that binary supports memory-to-memory processing */
1639 if (bi->info.isp.sp.input.source !=
1640 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1641 continue;
1642
1643 /* Check that binary supports raw10 input */
1644 if (!bi->info.isp.sp.enable.input_feeder &&
1645 !bi->info.isp.sp.enable.input_raw)
1646 continue;
1647
1648 /* Check binary mode */
1649 if (bi->info.isp.sp.pipeline.mode != binary_mode)
1650 continue;
1651
1652 /* Since input is RGGB bayer, need to process colors */
1653 if (bi->info.isp.sp.enable.luma_only)
1654 continue;
1655
1656 if (in->width < bi->info.isp.sp.input.min_width ||
1657 in->width > bi->info.isp.sp.input.max_width ||
1658 in->height < bi->info.isp.sp.input.min_height ||
1659 in->height > bi->info.isp.sp.input.max_height)
1660 continue;
1661
1662 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1663 if (bi->info.isp.num_output_pins <= 0)
1664 continue;
1665
1666 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1667 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1668 if (bi->info.isp.output_formats[j] == q_fmt)
1669 break;
1670 if (j >= bi->info.isp.num_output_formats)
1671 continue;
1672
1673 if (out->width < bi->info.isp.sp.output.min_width ||
1674 out->width > bi->info.isp.sp.output.max_width ||
1675 out->height < bi->info.isp.sp.output.min_height ||
1676 out->height > bi->info.isp.sp.output.max_height)
1677 continue;
1678
1679 if (out->width > bi->info.isp.sp.internal.max_width ||
1680 out->height > bi->info.isp.sp.internal.max_height)
1681 continue;
1682 }
1683
1684 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1685 if (bi->info.isp.num_output_pins <= 1)
1686 continue;
1687
1688 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1689 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1690 if (bi->info.isp.output_formats[j] == q_fmt)
1691 break;
1692 if (j >= bi->info.isp.num_output_formats)
1693 continue;
1694
1695 if (vf->width < bi->info.isp.sp.output.min_width ||
1696 vf->width > bi->info.isp.sp.output.max_width ||
1697 vf->height < bi->info.isp.sp.output.min_height ||
1698 vf->height > bi->info.isp.sp.output.max_height)
1699 continue;
1700 }
1701
1702 /* All checks passed, select the binary */
1703 dev_dbg(css->dev, "using binary %s id = %u\n", name,
1704 bi->info.isp.sp.id);
1705 return i;
1706 }
1707
1708 /* Can not find suitable binary for these parameters */
1709 return -EINVAL;
1710 }
1711
1712 /*
1713 * Check that there is a binary matching requirements. Parameters may be
1714 * NULL indicating disabled input/output. Return negative if given
1715 * parameters can not be supported or on error, zero or positive indicating
1716 * found binary number. May modify the given parameters if not exact match
1717 * is found.
1718 */
1719 int imgu_css_fmt_try(struct imgu_css *css,
1720 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1721 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1722 unsigned int pipe)
1723 {
1724 static const u32 EFF_ALIGN_W = 2;
1725 static const u32 BDS_ALIGN_W = 4;
1726 static const u32 OUT_ALIGN_W = 8;
1727 static const u32 OUT_ALIGN_H = 4;
1728 static const u32 VF_ALIGN_W = 2;
1729 static const char *qnames[IPU3_CSS_QUEUES] = {
1730 [IPU3_CSS_QUEUE_IN] = "in",
1731 [IPU3_CSS_QUEUE_PARAMS] = "params",
1732 [IPU3_CSS_QUEUE_OUT] = "out",
1733 [IPU3_CSS_QUEUE_VF] = "vf",
1734 [IPU3_CSS_QUEUE_STAT_3A] = "3a",
1735 };
1736 static const char *rnames[IPU3_CSS_RECTS] = {
1737 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1738 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution",
1739 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size",
1740 [IPU3_CSS_RECT_GDC] = "GDC output res",
1741 };
1742 struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1743 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1744 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1745 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1746 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1747 struct imgu_css_queue q[IPU3_CSS_QUEUES];
1748 struct v4l2_pix_format_mplane *const in =
1749 &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1750 struct v4l2_pix_format_mplane *const out =
1751 &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1752 struct v4l2_pix_format_mplane *const vf =
1753 &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1754 int i, s, ret;
1755
1756 /* Adjust all formats, get statistics buffer sizes and formats */
1757 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1758 if (fmts[i])
1759 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1760 qnames[i], fmts[i]->width, fmts[i]->height,
1761 fmts[i]->pixelformat);
1762 else
1763 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1764 qnames[i]);
1765 if (imgu_css_queue_init(&q[i], fmts[i],
1766 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1767 dev_notice(css->dev, "can not initialize queue %s\n",
1768 qnames[i]);
1769 return -EINVAL;
1770 }
1771 }
1772 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1773 if (rects[i]) {
1774 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1775 rnames[i], rects[i]->width, rects[i]->height);
1776 r[i].width = rects[i]->width;
1777 r[i].height = rects[i]->height;
1778 } else {
1779 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1780 rnames[i]);
1781 }
1782 /* For now, force known good resolutions */
1783 r[i].left = 0;
1784 r[i].top = 0;
1785 }
1786
1787 /* Always require one input and vf only if out is also enabled */
1788 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1789 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1790 dev_warn(css->dev, "required queues are disabled\n");
1791 return -EINVAL;
1792 }
1793
1794 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1795 out->width = in->width;
1796 out->height = in->height;
1797 }
1798 if (eff->width <= 0 || eff->height <= 0) {
1799 eff->width = in->width;
1800 eff->height = in->height;
1801 }
1802 if (bds->width <= 0 || bds->height <= 0) {
1803 bds->width = out->width;
1804 bds->height = out->height;
1805 }
1806 if (gdc->width <= 0 || gdc->height <= 0) {
1807 gdc->width = out->width;
1808 gdc->height = out->height;
1809 }
1810
1811 in->width = imgu_css_adjust(in->width, 1);
1812 in->height = imgu_css_adjust(in->height, 1);
1813 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1814 eff->height = imgu_css_adjust(eff->height, 1);
1815 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1816 bds->height = imgu_css_adjust(bds->height, 1);
1817 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1818 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1819 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W);
1820 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1821 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W);
1822 vf->height = imgu_css_adjust(vf->height, 1);
1823
1824 s = (bds->width - gdc->width) / 2 - FILTER_SIZE;
1825 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1826 s = (bds->height - gdc->height) / 2 - FILTER_SIZE;
1827 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1828
1829 ret = imgu_css_find_binary(css, pipe, q, r);
1830 if (ret < 0) {
1831 dev_err(css->dev, "failed to find suitable binary\n");
1832 return -EINVAL;
1833 }
1834 css->pipes[pipe].bindex = ret;
1835
1836 dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1837 css->pipes[pipe].bindex, pipe);
1838
1839 /* Final adjustment and set back the queried formats */
1840 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1841 if (fmts[i]) {
1842 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1843 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1844 dev_err(css->dev,
1845 "final resolution adjustment failed\n");
1846 return -EINVAL;
1847 }
1848 *fmts[i] = q[i].fmt.mpix;
1849 }
1850 }
1851
1852 for (i = 0; i < IPU3_CSS_RECTS; i++)
1853 if (rects[i])
1854 *rects[i] = r[i];
1855
1856 dev_dbg(css->dev,
1857 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1858 in->width, in->height, eff->width, eff->height,
1859 bds->width, bds->height, gdc->width, gdc->height,
1860 out->width, out->height, vf->width, vf->height);
1861
1862 return 0;
1863 }
1864
1865 int imgu_css_fmt_set(struct imgu_css *css,
1866 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1867 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1868 unsigned int pipe)
1869 {
1870 struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1871 struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1872 int i, r;
1873 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1874
1875 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1876 if (rects[i])
1877 rect_data[i] = *rects[i];
1878 else
1879 memset(&rect_data[i], 0, sizeof(rect_data[i]));
1880 all_rects[i] = &rect_data[i];
1881 }
1882 r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1883 if (r < 0)
1884 return r;
1885
1886 for (i = 0; i < IPU3_CSS_QUEUES; i++)
1887 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1888 IPU3_CSS_QUEUE_TO_FLAGS(i)))
1889 return -EINVAL;
1890 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1891 css_pipe->rect[i] = rect_data[i];
1892 if (rects[i])
1893 *rects[i] = rect_data[i];
1894 }
1895
1896 return 0;
1897 }
1898
1899 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1900 {
1901 switch (fmt->dataformat) {
1902 case V4L2_META_FMT_IPU3_PARAMS:
1903 fmt->buffersize = sizeof(struct ipu3_uapi_params);
1904 break;
1905 case V4L2_META_FMT_IPU3_STAT_3A:
1906 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1907 break;
1908 default:
1909 return -EINVAL;
1910 }
1911
1912 return 0;
1913 }
1914
1915 /*
1916 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1917 * called for the buffer. May be called from interrupt context.
1918 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1919 * code on error conditions.
1920 */
1921 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1922 struct imgu_css_buffer *b)
1923 {
1924 struct imgu_abi_buffer *abi_buf;
1925 struct imgu_addr_t *buf_addr;
1926 u32 data;
1927 int r;
1928 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1929
1930 if (!css->streaming)
1931 return -EPROTO; /* CSS or buffer in wrong state */
1932
1933 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1934 return -EINVAL;
1935
1936 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1937 pipe);
1938
1939 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1940 return -EIO;
1941 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1942
1943 /* Fill struct abi_buffer for firmware */
1944 memset(abi_buf, 0, sizeof(*abi_buf));
1945
1946 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1947 *(imgu_addr_t *)buf_addr = b->daddr;
1948
1949 if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1950 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1951
1952 if (b->queue == IPU3_CSS_QUEUE_OUT)
1953 abi_buf->payload.frame.padded_width =
1954 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1955
1956 if (b->queue == IPU3_CSS_QUEUE_VF)
1957 abi_buf->payload.frame.padded_width =
1958 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1959
1960 spin_lock(&css_pipe->qlock);
1961 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1962 spin_unlock(&css_pipe->qlock);
1963 b->state = IPU3_CSS_BUFFER_QUEUED;
1964
1965 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1966 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1967 pipe, data);
1968 if (r < 0)
1969 goto queueing_failed;
1970
1971 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1972 imgu_css_queues[b->queue].qid);
1973 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1974 if (r < 0)
1975 goto queueing_failed;
1976
1977 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1978 b, b->queue, pipe);
1979
1980 return 0;
1981
1982 queueing_failed:
1983 b->state = (r == -EBUSY || r == -EAGAIN) ?
1984 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1985 list_del(&b->list);
1986
1987 return r;
1988 }
1989
1990 /*
1991 * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1992 * should be called again, or -EBUSY which means that there are no more
1993 * buffers available. May be called from interrupt context.
1994 */
1995 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1996 {
1997 static const unsigned char evtype_to_queue[] = {
1998 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1999 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
2000 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
2001 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
2002 };
2003 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
2004 u32 event, daddr;
2005 int evtype, pipe, pipeid, queue, qid, r;
2006 struct imgu_css_pipe *css_pipe;
2007
2008 if (!css->streaming)
2009 return ERR_PTR(-EPROTO);
2010
2011 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2012 if (r < 0)
2013 return ERR_PTR(r);
2014
2015 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2016 IMGU_ABI_EVTTYPE_EVENT_SHIFT;
2017
2018 switch (evtype) {
2019 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
2020 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
2021 case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
2022 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
2023 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2024 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2025 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
2026 IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
2027 queue = evtype_to_queue[evtype];
2028 qid = imgu_css_queues[queue].qid;
2029
2030 if (pipe >= IMGU_MAX_PIPE_NUM) {
2031 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2032 return ERR_PTR(-EIO);
2033 }
2034
2035 if (qid >= IMGU_ABI_QUEUE_NUM) {
2036 dev_err(css->dev, "Invalid qid: %i\n", qid);
2037 return ERR_PTR(-EIO);
2038 }
2039 css_pipe = &css->pipes[pipe];
2040 dev_dbg(css->dev,
2041 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2042 event, queue, pipe, pipeid);
2043
2044 r = imgu_css_dequeue_data(css, qid, &daddr);
2045 if (r < 0) {
2046 dev_err(css->dev, "failed to dequeue buffer\n");
2047 /* Force real error, not -EBUSY */
2048 return ERR_PTR(-EIO);
2049 }
2050
2051 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2052 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2053 if (r < 0) {
2054 dev_err(css->dev, "failed to queue event\n");
2055 return ERR_PTR(-EIO);
2056 }
2057
2058 spin_lock(&css_pipe->qlock);
2059 if (list_empty(&css_pipe->queue[queue].bufs)) {
2060 spin_unlock(&css_pipe->qlock);
2061 dev_err(css->dev, "event on empty queue\n");
2062 return ERR_PTR(-EIO);
2063 }
2064 b = list_first_entry(&css_pipe->queue[queue].bufs,
2065 struct imgu_css_buffer, list);
2066 if (queue != b->queue ||
2067 daddr != css_pipe->abi_buffers
2068 [b->queue][b->queue_pos].daddr) {
2069 spin_unlock(&css_pipe->qlock);
2070 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2071 return ERR_PTR(-EIO);
2072 }
2073
2074 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2075 b->pipe = pipe;
2076 b->state = IPU3_CSS_BUFFER_DONE;
2077 list_del(&b->list);
2078 spin_unlock(&css_pipe->qlock);
2079 break;
2080 case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2081 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2082 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2083 if (pipe >= IMGU_MAX_PIPE_NUM) {
2084 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2085 return ERR_PTR(-EIO);
2086 }
2087
2088 css_pipe = &css->pipes[pipe];
2089 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2090 event, pipe);
2091 break;
2092 case IMGU_ABI_EVTTYPE_TIMER:
2093 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2094 if (r < 0)
2095 return ERR_PTR(r);
2096
2097 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2098 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2099 dev_dbg(css->dev, "event: timer\n");
2100 else
2101 dev_warn(css->dev, "half of timer event missing\n");
2102 break;
2103 case IMGU_ABI_EVTTYPE_FW_WARNING:
2104 dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2105 break;
2106 case IMGU_ABI_EVTTYPE_FW_ASSERT:
2107 dev_err(css->dev,
2108 "event: firmware assert 0x%x module_id %i line_no %i\n",
2109 event,
2110 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2111 IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2112 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2113 IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2114 break;
2115 default:
2116 dev_warn(css->dev, "received unknown event 0x%x\n", event);
2117 }
2118
2119 return b;
2120 }
2121
2122 /*
2123 * Get a new set of parameters from pool and initialize them based on
2124 * the parameters params, gdc, and obgrid. Any of these may be NULL,
2125 * in which case the previously set parameters are used.
2126 * If parameters haven't been set previously, initialize from scratch.
2127 *
2128 * Return index to css->parameter_set_info which has the newly created
2129 * parameters or negative value on error.
2130 */
2131 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2132 struct ipu3_uapi_params *set_params)
2133 {
2134 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2135 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2136 const int stage = 0;
2137 const struct imgu_fw_info *bi;
2138 int obgrid_size;
2139 unsigned int stripes, i;
2140 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2141
2142 /* Destination buffers which are filled here */
2143 struct imgu_abi_parameter_set_info *param_set;
2144 struct imgu_abi_acc_param *acc = NULL;
2145 struct imgu_abi_gdc_warp_param *gdc = NULL;
2146 struct ipu3_uapi_obgrid_param *obgrid = NULL;
2147 const struct imgu_css_map *map;
2148 void *vmem0 = NULL;
2149 void *dmem0 = NULL;
2150
2151 enum imgu_abi_memories m;
2152 int r = -EBUSY;
2153
2154 if (!css->streaming)
2155 return -EPROTO;
2156
2157 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2158
2159 bi = &css->fwp->binary_header[css_pipe->bindex];
2160 obgrid_size = imgu_css_fw_obgrid_size(bi);
2161 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2162
2163 /*
2164 * TODO(b/118782861): If userspace queues more than 4 buffers, the
2165 * parameters from previous buffers will be overwritten. Fix the driver
2166 * not to allow this.
2167 */
2168 imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2169 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2170 0)->vaddr;
2171
2172 /* Get a new acc only if new parameters given, or none yet */
2173 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2174 if (set_params || !map->vaddr) {
2175 imgu_css_pool_get(&css_pipe->pool.acc);
2176 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2177 acc = map->vaddr;
2178 }
2179
2180 /* Get new VMEM0 only if needed, or none yet */
2181 m = IMGU_ABI_MEM_ISP_VMEM0;
2182 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2183 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2184 set_params->use.tnr3_vmem_params ||
2185 set_params->use.xnr3_vmem_params))) {
2186 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2187 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2188 vmem0 = map->vaddr;
2189 }
2190
2191 /* Get new DMEM0 only if needed, or none yet */
2192 m = IMGU_ABI_MEM_ISP_DMEM0;
2193 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2194 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2195 set_params->use.xnr3_dmem_params))) {
2196 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2197 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2198 dmem0 = map->vaddr;
2199 }
2200
2201 /* Configure acc parameter cluster */
2202 if (acc) {
2203 /* get acc_old */
2204 map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2205 /* user acc */
2206 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2207 set_params ? &set_params->acc_param : NULL);
2208 if (r < 0)
2209 goto fail;
2210 }
2211
2212 /* Configure late binding parameters */
2213 if (vmem0) {
2214 m = IMGU_ABI_MEM_ISP_VMEM0;
2215 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2216 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2217 map->vaddr, set_params);
2218 if (r < 0)
2219 goto fail;
2220 }
2221
2222 if (dmem0) {
2223 m = IMGU_ABI_MEM_ISP_DMEM0;
2224 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2225 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2226 map->vaddr, set_params);
2227 if (r < 0)
2228 goto fail;
2229 }
2230
2231 /* Get a new gdc only if a new gdc is given, or none yet */
2232 if (bi->info.isp.sp.enable.dvs_6axis) {
2233 unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2234 unsigned int g = IPU3_CSS_RECT_GDC;
2235 unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2236
2237 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2238 if (!map->vaddr) {
2239 imgu_css_pool_get(&css_pipe->pool.gdc);
2240 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2241 gdc = map->vaddr;
2242 imgu_css_cfg_gdc_table(map->vaddr,
2243 css_pipe->aux_frames[a].bytesperline /
2244 css_pipe->aux_frames[a].bytesperpixel,
2245 css_pipe->aux_frames[a].height,
2246 css_pipe->rect[g].width,
2247 css_pipe->rect[g].height,
2248 css_pipe->rect[e].width + FILTER_SIZE,
2249 css_pipe->rect[e].height +
2250 FILTER_SIZE);
2251 }
2252 }
2253
2254 /* Get a new obgrid only if a new obgrid is given, or none yet */
2255 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2256 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2257 imgu_css_pool_get(&css_pipe->pool.obgrid);
2258 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2259 obgrid = map->vaddr;
2260
2261 /* Configure optical black level grid (obgrid) */
2262 if (set_params && set_params->use.obgrid_param)
2263 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2264 obgrid[i] = set_params->obgrid_param;
2265 else
2266 memset(obgrid, 0, obgrid_size);
2267 }
2268
2269 /* Configure parameter set info, queued to `queue_id' */
2270
2271 memset(param_set, 0, sizeof(*param_set));
2272 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2273 param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2274
2275 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2276 param_set->mem_map.dvs_6axis_params_y = map->daddr;
2277
2278 for (i = 0; i < stripes; i++) {
2279 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2280 param_set->mem_map.obgrid_tbl[i] =
2281 map->daddr + (obgrid_size / stripes) * i;
2282 }
2283
2284 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2285 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2286 param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2287 }
2288
2289 /* Then queue the new parameter buffer */
2290 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2291 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2292 if (r < 0)
2293 goto fail;
2294
2295 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2296 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2297 queue_id));
2298 if (r < 0)
2299 goto fail_no_put;
2300
2301 /* Finally dequeue all old parameter buffers */
2302
2303 do {
2304 u32 daddr;
2305
2306 r = imgu_css_dequeue_data(css, queue_id, &daddr);
2307 if (r == -EBUSY)
2308 break;
2309 if (r)
2310 goto fail_no_put;
2311 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2312 IMGU_ABI_EVENT_BUFFER_DEQUEUED
2313 (queue_id));
2314 if (r < 0) {
2315 dev_err(css->dev, "failed to queue parameter event\n");
2316 goto fail_no_put;
2317 }
2318 } while (1);
2319
2320 return 0;
2321
2322 fail:
2323 /*
2324 * A failure, most likely the parameter queue was full.
2325 * Return error but continue streaming. User can try submitting new
2326 * parameters again later.
2327 */
2328
2329 imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2330 if (acc)
2331 imgu_css_pool_put(&css_pipe->pool.acc);
2332 if (gdc)
2333 imgu_css_pool_put(&css_pipe->pool.gdc);
2334 if (obgrid)
2335 imgu_css_pool_put(&css_pipe->pool.obgrid);
2336 if (vmem0)
2337 imgu_css_pool_put(
2338 &css_pipe->pool.binary_params_p
2339 [IMGU_ABI_MEM_ISP_VMEM0]);
2340 if (dmem0)
2341 imgu_css_pool_put(
2342 &css_pipe->pool.binary_params_p
2343 [IMGU_ABI_MEM_ISP_DMEM0]);
2344
2345 fail_no_put:
2346 return r;
2347 }
2348
2349 int imgu_css_irq_ack(struct imgu_css *css)
2350 {
2351 static const int NUM_SWIRQS = 3;
2352 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2353 void __iomem *const base = css->base;
2354 u32 irq_status[IMGU_IRQCTRL_NUM];
2355 int i;
2356
2357 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2358
2359 writel(imgu_status, base + IMGU_REG_INT_STATUS);
2360 for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2361 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2362
2363 for (i = 0; i < NUM_SWIRQS; i++) {
2364 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2365 /* SP SW interrupt */
2366 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2367 bi->info.sp.output);
2368 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2369 bi->info.sp.output + 4 + 4 * i);
2370
2371 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2372 __func__, i, cnt, val);
2373 }
2374 }
2375
2376 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2377 if (irq_status[i]) {
2378 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2379 /* Wait for write to complete */
2380 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2381 }
2382
2383 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2384 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2385 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2386
2387 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2388 return -ENOMSG;
2389
2390 return 0;
2391 }
Ubuntu combined Linux kernel mirror