[mirror_qemu.git] / ui / vnc-enc-hextile-template.h

#define CONCAT_I(a, b) a ## b
#define CONCAT(a, b) CONCAT_I(a, b)
#define pixel_t CONCAT(uint, CONCAT(BPP, _t))
#ifdef GENERIC
#define NAME CONCAT(generic_, BPP)
#else
#define NAME BPP
#endif

#define MAX_BYTES_PER_PIXEL 4

static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
                                             int x, int y, int w, int h,
                                             void *last_bg_,
                                             void *last_fg_,
                                             int *has_bg, int *has_fg)
{
    VncDisplay *vd = vs->vd;
    uint8_t *row = vnc_server_fb_ptr(vd, x, y);
    pixel_t *irow = (pixel_t *)row;
    int j, i;
    pixel_t *last_bg = (pixel_t *)last_bg_;
    pixel_t *last_fg = (pixel_t *)last_fg_;
    pixel_t bg = 0;
    pixel_t fg = 0;
    int n_colors = 0;
    int bg_count = 0;
    int fg_count = 0;
    int flags = 0;
    uint8_t data[(MAX_BYTES_PER_PIXEL + 2) * 16 * 16];
    int n_data = 0;
    int n_subtiles = 0;

    /* Enforced by set_pixel_format() */
    assert(vs->client_pf.bytes_per_pixel <= MAX_BYTES_PER_PIXEL);

    for (j = 0; j < h; j++) {
        for (i = 0; i < w; i++) {
            switch (n_colors) {
            case 0:
                bg = irow[i];
                n_colors = 1;
                break;
            case 1:
                if (irow[i] != bg) {
                    fg = irow[i];
                    n_colors = 2;
                }
                break;
            case 2:
                if (irow[i] != bg && irow[i] != fg) {
                    n_colors = 3;
                } else {
                    if (irow[i] == bg)
                        bg_count++;
                    else if (irow[i] == fg)
                        fg_count++;
                }
                break;
            default:
                break;
            }
        }
        if (n_colors > 2)
            break;
        irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
    }

    if (n_colors > 1 && fg_count > bg_count) {
        pixel_t tmp = fg;
        fg = bg;
        bg = tmp;
    }

    if (!*has_bg || *last_bg != bg) {
        flags |= 0x02;
        *has_bg = 1;
        *last_bg = bg;
    }

    if (n_colors < 3 && (!*has_fg || *last_fg != fg)) {
        flags |= 0x04;
        *has_fg = 1;
        *last_fg = fg;
    }

    switch (n_colors) {
    case 1:
        n_data = 0;
        break;
    case 2:
        flags |= 0x08;

        irow = (pixel_t *)row;

        for (j = 0; j < h; j++) {
            int min_x = -1;
            for (i = 0; i < w; i++) {
                if (irow[i] == fg) {
                    if (min_x == -1)
                        min_x = i;
                } else if (min_x != -1) {
                    hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
                    n_data += 2;
                    n_subtiles++;
                    min_x = -1;
                }
            }
            if (min_x != -1) {
                hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
                n_data += 2;
                n_subtiles++;
            }
            irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
        }
        break;
    case 3:
        flags |= 0x18;

        irow = (pixel_t *)row;

        if (!*has_bg || *last_bg != bg)
            flags |= 0x02;

        for (j = 0; j < h; j++) {
            int has_color = 0;
            int min_x = -1;
            pixel_t color = 0; /* shut up gcc */

            for (i = 0; i < w; i++) {
                if (!has_color) {
                    if (irow[i] == bg)
                        continue;
                    color = irow[i];
                    min_x = i;
                    has_color = 1;
                } else if (irow[i] != color) {
                    has_color = 0;
#ifdef GENERIC
                    vnc_convert_pixel(vs, data + n_data, color);
                    n_data += vs->client_pf.bytes_per_pixel;
#else
                    memcpy(data + n_data, &color, sizeof(color));
                    n_data += sizeof(pixel_t);
#endif
                    hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
                    n_data += 2;
                    n_subtiles++;

                    min_x = -1;
                    if (irow[i] != bg) {
                        color = irow[i];
                        min_x = i;
                        has_color = 1;
                    }
                }
            }
            if (has_color) {
#ifdef GENERIC
                vnc_convert_pixel(vs, data + n_data, color);
                n_data += vs->client_pf.bytes_per_pixel;
#else
                memcpy(data + n_data, &color, sizeof(color));
                n_data += sizeof(pixel_t);
#endif
                hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
                n_data += 2;
                n_subtiles++;
            }
            irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
        }

        /* A SubrectsColoured subtile invalidates the foreground color */
        *has_fg = 0;
        if (n_data > (w * h * sizeof(pixel_t))) {
            n_colors = 4;
            flags = 0x01;
            *has_bg = 0;

            /* we really don't have to invalidate either the bg or fg
               but we've lost the old values.  oh well. */
        }
        break;
    default:
        break;
    }

    if (n_colors > 3) {
        flags = 0x01;
        *has_fg = 0;
        *has_bg = 0;
        n_colors = 4;
    }

    vnc_write_u8(vs, flags);
    if (n_colors < 4) {
        if (flags & 0x02)
            vs->write_pixels(vs, last_bg, sizeof(pixel_t));
        if (flags & 0x04)
            vs->write_pixels(vs, last_fg, sizeof(pixel_t));
        if (n_subtiles) {
            vnc_write_u8(vs, n_subtiles);
            vnc_write(vs, data, n_data);
        }
    } else {
        for (j = 0; j < h; j++) {
            vs->write_pixels(vs, row, w * 4);
            row += vnc_server_fb_stride(vd);
        }
    }
}

#undef MAX_BYTES_PER_PIXEL
#undef NAME
#undef pixel_t
#undef CONCAT_I
#undef CONCAT
Commit	Line	Data
24236869 FB	1	#define CONCAT_I(a, b) a ## b
	2	#define CONCAT(a, b) CONCAT_I(a, b)
	3	#define pixel_t CONCAT(uint, CONCAT(BPP, _t))
3512779a	4	#ifdef GENERIC
7eac3a87	5	#define NAME CONCAT(generic_, BPP)
3512779a FB	6	#else
	7	#define NAME BPP
	8	#endif
	9
e12acaf7 PM	10	#define MAX_BYTES_PER_PIXEL 4
e12acaf7 PM	11
3512779a FB	12	static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
3512779a FB	13	int x, int y, int w, int h,
7eac3a87 AL	14	void *last_bg_,
7eac3a87 AL	15	void *last_fg_,
3512779a	16	int has_bg, int has_fg)
24236869	17	{
1fc62412	18	VncDisplay *vd = vs->vd;
9f64916d	19	uint8_t *row = vnc_server_fb_ptr(vd, x, y);
24236869 FB	20	pixel_t irow = (pixel_t )row;
24236869 FB	21	int j, i;
7eac3a87 AL	22	pixel_t last_bg = (pixel_t )last_bg_;
7eac3a87 AL	23	pixel_t last_fg = (pixel_t )last_fg_;
24236869 FB	24	pixel_t bg = 0;
	25	pixel_t fg = 0;
	26	int n_colors = 0;
	27	int bg_count = 0;
	28	int fg_count = 0;
	29	int flags = 0;
e12acaf7	30	uint8_t data[(MAX_BYTES_PER_PIXEL + 2) * 16 * 16];
24236869 FB	31	int n_data = 0;
	32	int n_subtiles = 0;
	33
e12acaf7 PM	34	/* Enforced by set_pixel_format() */
	35	assert(vs->client_pf.bytes_per_pixel <= MAX_BYTES_PER_PIXEL);
	36
24236869	37	for (j = 0; j < h; j++) {
b86d01ba PB	38	for (i = 0; i < w; i++) {
	39	switch (n_colors) {
	40	case 0:
	41	bg = irow[i];
	42	n_colors = 1;
	43	break;
	44	case 1:
	45	if (irow[i] != bg) {
	46	fg = irow[i];
	47	n_colors = 2;
	48	}
	49	break;
	50	case 2:
	51	if (irow[i] != bg && irow[i] != fg) {
	52	n_colors = 3;
	53	} else {
	54	if (irow[i] == bg)
	55	bg_count++;
	56	else if (irow[i] == fg)
	57	fg_count++;
	58	}
	59	break;
	60	default:
	61	break;
	62	}
	63	}
	64	if (n_colors > 2)
	65	break;
	66	irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
24236869 FB	67	}
	68
	69	if (n_colors > 1 && fg_count > bg_count) {
b86d01ba PB	70	pixel_t tmp = fg;
	71	fg = bg;
	72	bg = tmp;
24236869 FB	73	}
	74
	75	if (!has_bg \|\| last_bg != bg) {
b86d01ba PB	76	flags \|= 0x02;
	77	*has_bg = 1;
	78	*last_bg = bg;
24236869 FB	79	}
24236869 FB	80
02c2b87f	81	if (n_colors < 3 && (!has_fg \|\| last_fg != fg)) {
b86d01ba PB	82	flags \|= 0x04;
	83	*has_fg = 1;
	84	*last_fg = fg;
24236869 FB	85	}
	86
	87	switch (n_colors) {
	88	case 1:
b86d01ba PB	89	n_data = 0;
b86d01ba PB	90	break;
24236869	91	case 2:
b86d01ba PB	92	flags \|= 0x08;
	93
	94	irow = (pixel_t *)row;
	95
	96	for (j = 0; j < h; j++) {
	97	int min_x = -1;
	98	for (i = 0; i < w; i++) {
	99	if (irow[i] == fg) {
	100	if (min_x == -1)
	101	min_x = i;
	102	} else if (min_x != -1) {
	103	hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
	104	n_data += 2;
	105	n_subtiles++;
	106	min_x = -1;
	107	}
	108	}
	109	if (min_x != -1) {
	110	hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
	111	n_data += 2;
	112	n_subtiles++;
	113	}
	114	irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
	115	}
	116	break;
24236869	117	case 3:
b86d01ba PB	118	flags \|= 0x18;
	119
	120	irow = (pixel_t *)row;
	121
	122	if (!has_bg \|\| last_bg != bg)
	123	flags \|= 0x02;
	124
	125	for (j = 0; j < h; j++) {
	126	int has_color = 0;
	127	int min_x = -1;
	128	pixel_t color = 0; /* shut up gcc */
	129
	130	for (i = 0; i < w; i++) {
	131	if (!has_color) {
	132	if (irow[i] == bg)
	133	continue;
	134	color = irow[i];
	135	min_x = i;
	136	has_color = 1;
	137	} else if (irow[i] != color) {
	138	has_color = 0;
3512779a FB	139	#ifdef GENERIC
3512779a FB	140	vnc_convert_pixel(vs, data + n_data, color);
9f64916d	141	n_data += vs->client_pf.bytes_per_pixel;
3512779a	142	#else
b86d01ba	143	memcpy(data + n_data, &color, sizeof(color));
3512779a FB	144	n_data += sizeof(pixel_t);
3512779a FB	145	#endif
b86d01ba PB	146	hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
	147	n_data += 2;
	148	n_subtiles++;
	149
	150	min_x = -1;
	151	if (irow[i] != bg) {
	152	color = irow[i];
	153	min_x = i;
	154	has_color = 1;
	155	}
	156	}
	157	}
	158	if (has_color) {
3512779a FB	159	#ifdef GENERIC
3512779a FB	160	vnc_convert_pixel(vs, data + n_data, color);
9f64916d	161	n_data += vs->client_pf.bytes_per_pixel;
3512779a FB	162	#else
	163	memcpy(data + n_data, &color, sizeof(color));
	164	n_data += sizeof(pixel_t);
	165	#endif
b86d01ba PB	166	hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
	167	n_data += 2;
	168	n_subtiles++;
	169	}
	170	irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
	171	}
	172
	173	/* A SubrectsColoured subtile invalidates the foreground color */
	174	*has_fg = 0;
	175	if (n_data > (w * h * sizeof(pixel_t))) {
	176	n_colors = 4;
	177	flags = 0x01;
	178	*has_bg = 0;
	179
	180	/* we really don't have to invalidate either the bg or fg
	181	but we've lost the old values. oh well. */
	182	}
0ea5c0cd	183	break;
24236869	184	default:
b86d01ba	185	break;
24236869 FB	186	}
	187
	188	if (n_colors > 3) {
b86d01ba PB	189	flags = 0x01;
	190	*has_fg = 0;
	191	*has_bg = 0;
	192	n_colors = 4;
24236869 FB	193	}
	194
	195	vnc_write_u8(vs, flags);
	196	if (n_colors < 4) {
b86d01ba PB	197	if (flags & 0x02)
	198	vs->write_pixels(vs, last_bg, sizeof(pixel_t));
	199	if (flags & 0x04)
	200	vs->write_pixels(vs, last_fg, sizeof(pixel_t));
	201	if (n_subtiles) {
	202	vnc_write_u8(vs, n_subtiles);
	203	vnc_write(vs, data, n_data);
	204	}
24236869	205	} else {
b86d01ba PB	206	for (j = 0; j < h; j++) {
	207	vs->write_pixels(vs, row, w * 4);
	208	row += vnc_server_fb_stride(vd);
	209	}
24236869 FB	210	}
	211	}
	212
e12acaf7	213	#undef MAX_BYTES_PER_PIXEL
3512779a	214	#undef NAME
24236869 FB	215	#undef pixel_t
	216	#undef CONCAT_I
	217	#undef CONCAT