]> git.proxmox.com Git - mirror_qemu.git/blame - ui/vnc-enc-zlib.c
target/s390x: Fix assertion failure in VFMIN/VFMAX with type 13
[mirror_qemu.git] / ui / vnc-enc-zlib.c
CommitLineData
70a4568f
CC
1/*
2 * QEMU VNC display driver: zlib encoding
3 *
4 * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
5 * Copyright (C) 2006 Fabrice Bellard
6 * Copyright (C) 2009 Red Hat, Inc
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 */
26
e16f4c87 27#include "qemu/osdep.h"
70a4568f
CC
28#include "vnc.h"
29
30#define ZALLOC_ALIGNMENT 16
31
380282b0 32void *vnc_zlib_zalloc(void *x, unsigned items, unsigned size)
70a4568f
CC
33{
34 void *p;
35
36 size *= items;
37 size = (size + ZALLOC_ALIGNMENT - 1) & ~(ZALLOC_ALIGNMENT - 1);
38
7267c094 39 p = g_malloc0(size);
70a4568f
CC
40
41 return (p);
42}
43
380282b0 44void vnc_zlib_zfree(void *x, void *addr)
70a4568f 45{
7267c094 46 g_free(addr);
70a4568f
CC
47}
48
49static void vnc_zlib_start(VncState *vs)
50{
d1af0e05 51 buffer_reset(&vs->zlib.zlib);
70a4568f
CC
52
53 // make the output buffer be the zlib buffer, so we can compress it later
d1af0e05
CC
54 vs->zlib.tmp = vs->output;
55 vs->output = vs->zlib.zlib;
70a4568f
CC
56}
57
b05ad290 58static int vnc_zlib_stop(VncState *vs)
70a4568f 59{
d1af0e05 60 z_streamp zstream = &vs->zlib.stream;
70a4568f
CC
61 int previous_out;
62
63 // switch back to normal output/zlib buffers
d1af0e05
CC
64 vs->zlib.zlib = vs->output;
65 vs->output = vs->zlib.tmp;
70a4568f
CC
66
67 // compress the zlib buffer
68
69 // initialize the stream
70 // XXX need one stream per session
71 if (zstream->opaque != vs) {
72 int err;
73
b05ad290 74 VNC_DEBUG("VNC: initializing zlib stream\n");
70a4568f 75 VNC_DEBUG("VNC: opaque = %p | vs = %p\n", zstream->opaque, vs);
380282b0
CC
76 zstream->zalloc = vnc_zlib_zalloc;
77 zstream->zfree = vnc_zlib_zfree;
70a4568f 78
6bf21f3d
LQ
79 err = deflateInit2(zstream, vs->tight->compression, Z_DEFLATED,
80 MAX_WBITS,
70a4568f
CC
81 MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
82
83 if (err != Z_OK) {
84 fprintf(stderr, "VNC: error initializing zlib\n");
85 return -1;
86 }
87
6bf21f3d 88 vs->zlib.level = vs->tight->compression;
70a4568f
CC
89 zstream->opaque = vs;
90 }
91
6bf21f3d
LQ
92 if (vs->tight->compression != vs->zlib.level) {
93 if (deflateParams(zstream, vs->tight->compression,
9f643ec0
CC
94 Z_DEFAULT_STRATEGY) != Z_OK) {
95 return -1;
96 }
6bf21f3d 97 vs->zlib.level = vs->tight->compression;
9f643ec0 98 }
70a4568f
CC
99
100 // reserve memory in output buffer
d1af0e05 101 buffer_reserve(&vs->output, vs->zlib.zlib.offset + 64);
70a4568f
CC
102
103 // set pointers
d1af0e05
CC
104 zstream->next_in = vs->zlib.zlib.buffer;
105 zstream->avail_in = vs->zlib.zlib.offset;
70a4568f
CC
106 zstream->next_out = vs->output.buffer + vs->output.offset;
107 zstream->avail_out = vs->output.capacity - vs->output.offset;
2caa9e9d 108 previous_out = zstream->avail_out;
70a4568f 109 zstream->data_type = Z_BINARY;
70a4568f
CC
110
111 // start encoding
112 if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
113 fprintf(stderr, "VNC: error during zlib compression\n");
114 return -1;
115 }
116
117 vs->output.offset = vs->output.capacity - zstream->avail_out;
2caa9e9d 118 return previous_out - zstream->avail_out;
70a4568f
CC
119}
120
a885211e 121int vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
70a4568f
CC
122{
123 int old_offset, new_offset, bytes_written;
124
125 vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_ZLIB);
126
127 // remember where we put in the follow-up size
128 old_offset = vs->output.offset;
129 vnc_write_s32(vs, 0);
130
131 // compress the stream
132 vnc_zlib_start(vs);
133 vnc_raw_send_framebuffer_update(vs, x, y, w, h);
b05ad290 134 bytes_written = vnc_zlib_stop(vs);
70a4568f
CC
135
136 if (bytes_written == -1)
a885211e 137 return 0;
70a4568f
CC
138
139 // hack in the size
140 new_offset = vs->output.offset;
141 vs->output.offset = old_offset;
142 vnc_write_u32(vs, bytes_written);
143 vs->output.offset = new_offset;
a885211e
CC
144
145 return 1;
70a4568f 146}
161c4f20
CC
147
148void vnc_zlib_clear(VncState *vs)
149{
d1af0e05
CC
150 if (vs->zlib.stream.opaque) {
151 deflateEnd(&vs->zlib.stream);
161c4f20 152 }
d1af0e05 153 buffer_free(&vs->zlib.zlib);
161c4f20 154}