]> git.proxmox.com Git - mirror_qemu.git/blobdiff - ui/vnc-enc-tight.c
iotests: Fix test 200 on s390x without virtio-pci
[mirror_qemu.git] / ui / vnc-enc-tight.c
index e6966aebc355a826ae08c51d5174fec97d02106c..d20cd1d86dfe15855b726e6c68110113d004117a 100644 (file)
@@ -26,7 +26,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 /* This needs to be before jpeglib.h line because of conflict with
    INT32 definitions between jmorecfg.h (included by jpeglib.h) and
 #include <png.h>
 #endif
 #ifdef CONFIG_VNC_JPEG
-#include <stdio.h>
 #include <jpeglib.h>
 #endif
 
 #include "qemu/bswap.h"
-#include "qapi/qmp/qint.h"
 #include "vnc.h"
 #include "vnc-enc-tight.h"
 #include "vnc-palette.h"
@@ -181,6 +179,10 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
         }
     }
 
+    if (pixels == 0) {
+        return 0;
+    }
+
     /* 95% smooth or more ... */
     if (stats[0] * 33 / pixels >= 95) {
         return 0;
@@ -216,8 +218,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
         unsigned int errors;                                            \
         unsigned char *buf = vs->tight.tight.buffer;                    \
                                                                         \
-        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
-                      (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
+        endian = 0; /* FIXME */                                         \
                                                                         \
                                                                         \
         max[0] = vs->client_pf.rmax;                                  \
@@ -267,7 +268,9 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
                 y += w;                                                 \
             }                                                           \
         }                                                               \
-                                                                        \
+        if (pixels == 0) {                                              \
+            return 0;                                                   \
+        }                                                               \
         if ((stats[0] + stats[1]) * 100 / pixels >= 90) {               \
             return 0;                                                   \
         }                                                               \
@@ -330,7 +333,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
     } else {
         errors = tight_detect_smooth_image16(vs, w, h);
     }
-    if (quality != -1) {
+    if (quality != (uint8_t)-1) {
         return (errors < tight_conf[quality].jpeg_threshold);
     }
     return (errors < tight_conf[compression].gradient_threshold);
@@ -345,7 +348,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
     tight_fill_palette##bpp(VncState *vs, int x, int y,                 \
                             int max, size_t count,                      \
                             uint32_t *bg, uint32_t *fg,                 \
-                            VncPalette **palette) {                     \
+                            VncPalette *palette) {                      \
         uint##bpp##_t *data;                                            \
         uint##bpp##_t c0, c1, ci;                                       \
         int i, n0, n1;                                                  \
@@ -392,23 +395,23 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
             return 0;                                                   \
         }                                                               \
                                                                         \
-        *palette = palette_new(max, bpp);                               \
-        palette_put(*palette, c0);                                      \
-        palette_put(*palette, c1);                                      \
-        palette_put(*palette, ci);                                      \
+        palette_init(palette, max, bpp);                                \
+        palette_put(palette, c0);                                       \
+        palette_put(palette, c1);                                       \
+        palette_put(palette, ci);                                       \
                                                                         \
         for (i++; i < count; i++) {                                     \
             if (data[i] == ci) {                                        \
                 continue;                                               \
             } else {                                                    \
                 ci = data[i];                                           \
-                if (!palette_put(*palette, (uint32_t)ci)) {             \
+                if (!palette_put(palette, (uint32_t)ci)) {              \
                     return 0;                                           \
                 }                                                       \
             }                                                           \
         }                                                               \
                                                                         \
-        return palette_size(*palette);                                  \
+        return palette_size(palette);                                   \
     }
 
 DEFINE_FILL_PALETTE_FUNCTION(8)
@@ -417,7 +420,7 @@ DEFINE_FILL_PALETTE_FUNCTION(32)
 
 static int tight_fill_palette(VncState *vs, int x, int y,
                               size_t count, uint32_t *bg, uint32_t *fg,
-                              VncPalette **palette)
+                              VncPalette *palette)
 {
     int max;
 
@@ -457,9 +460,10 @@ static int tight_fill_palette(VncState *vs, int x, int y,
                                                                         \
         src = (uint##bpp##_t *) buf;                                    \
                                                                         \
-        for (i = 0; i < count; i++) {                                   \
+        for (i = 0; i < count; ) {                                      \
                                                                         \
             rgb = *src++;                                               \
+            i++;                                                        \
             rep = 0;                                                    \
             while (i < count && *src == rgb) {                          \
                 rep++, src++, i++;                                      \
@@ -557,8 +561,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
     buf32 = (uint32_t *)buf;
     memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
 
-    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+    if (1 /* FIXME */) {
         shift[0] = vs->client_pf.rshift;
         shift[1] = vs->client_pf.gshift;
         shift[2] = vs->client_pf.bshift;
@@ -615,8 +618,7 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
                                                                         \
         memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));     \
                                                                         \
-        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
-                       (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
+        endian = 0; /* FIXME */                                         \
                                                                         \
         max[0] = vs->client_pf.rmax;                                    \
         max[1] = vs->client_pf.gmax;                                    \
@@ -704,10 +706,8 @@ check_solid_tile32(VncState *vs, int x, int y, int w, int h,
 static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,
                              uint32_t* color, bool samecolor)
 {
-    switch (VNC_SERVER_FB_BYTES) {
-    case 4:
-        return check_solid_tile32(vs, x, y, w, h, color, samecolor);
-    }
+    QEMU_BUILD_BUG_ON(VNC_SERVER_FB_BYTES != 4);
+    return check_solid_tile32(vs, x, y, w, h, color, samecolor);
 }
 
 static void find_best_solid_area(VncState *vs, int x, int y, int w, int h,
@@ -886,14 +886,13 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
  */
 static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
 {
-    uint32_t *buf32;
+    uint8_t *buf8;
     uint32_t pix;
     int rshift, gshift, bshift;
 
-    buf32 = (uint32_t *)buf;
+    buf8 = buf;
 
-    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+    if (1 /* FIXME */) {
         rshift = vs->client_pf.rshift;
         gshift = vs->client_pf.gshift;
         bshift = vs->client_pf.bshift;
@@ -908,10 +907,11 @@ static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
     }
 
     while (count--) {
-        pix = *buf32++;
+        pix = ldl_he_p(buf8);
         *buf++ = (char)(pix >> rshift);
         *buf++ = (char)(pix >> gshift);
         *buf++ = (char)(pix >> bshift);
+        buf8 += 4;
     }
 }
 
@@ -980,7 +980,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
     }
 #endif
 
-    bytes = ((w + 7) / 8) * h;
+    bytes = DIV_ROUND_UP(w, 8) * h;
 
     vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
     vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
@@ -1456,9 +1456,17 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
 }
 #endif
 
+static __thread VncPalette *color_count_palette;
+static __thread Notifier vnc_tight_cleanup_notifier;
+
+static void vnc_tight_cleanup(Notifier *n, void *value)
+{
+    g_free(color_count_palette);
+    color_count_palette = NULL;
+}
+
 static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
 {
-    VncPalette *palette = NULL;
     uint32_t bg = 0, fg = 0;
     int colors;
     int ret = 0;
@@ -1467,6 +1475,12 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
     bool allow_jpeg = true;
 #endif
 
+    if (!color_count_palette) {
+        color_count_palette = g_malloc(sizeof(VncPalette));
+        vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup;
+        qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
+    }
+
     vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
 
     vnc_tight_start(vs);
@@ -1487,20 +1501,21 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
     }
 #endif
 
-    colors = tight_fill_palette(vs, x, y, w * h, &fg, &bg, &palette);
+    colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette);
 
 #ifdef CONFIG_VNC_JPEG
     if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
-        ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette,
-                                 force_jpeg);
+        ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
+                                 color_count_palette, force_jpeg);
     } else {
-        ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+        ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+                                   color_count_palette);
     }
 #else
-    ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+    ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+                               color_count_palette);
 #endif
 
-    palette_destroy(palette);
     return ret;
 }