ui/vnc: fix vmware VGA incompatiblities

this fixes invalid rectangle updates observed after commit 12b316d
with the vmware VGA driver. The issues occured because the server
and client surface update seems to be out of sync at some points
and the max width of the surface is not dividable by
VNC_DIRTY_BITS_PER_PIXEL (16).

Reported-by: Serge Hallyn <serge.hallyn@ubuntu.com>
Signed-off-by: Peter Lieven <pl@kamp.de>
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

diff --git a/hw/display/vmware_vga.c b/hw/display/vmware_vga.c
index bd2c108c42f29fdc73fd9a8c3f291264095f75a2..6ae3348deb057ca229d7fdb4d55f585fa633d53e 100644 (file)
--- a/hw/display/vmware_vga.c
+++ b/hw/display/vmware_vga.c
@@ -25,6 +25,7 @@
 #include "hw/loader.h"
 #include "trace.h"
 #include "ui/console.h"
+#include "ui/vnc.h"
 #include "hw/pci/pci.h"
 
 #undef VERBOSE
@@ -218,7 +219,7 @@ enum {
 
 /* These values can probably be changed arbitrarily.  */
 #define SVGA_SCRATCH_SIZE               0x8000
-#define SVGA_MAX_WIDTH                  2360
+#define SVGA_MAX_WIDTH                  ROUND_UP(2360, VNC_DIRTY_PIXELS_PER_BIT)
 #define SVGA_MAX_HEIGHT                 1770
 
 #ifdef VERBOSE

diff --git a/ui/vnc.c b/ui/vnc.c
index 9c84b3e0fd9407d9a1226993098196b1cf51a911..59257745094d5fcd6ed2870bd09d19d3d1005aa0 100644 (file)
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -888,7 +888,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         VncDisplay *vd = vs->vd;
         VncJob *job;
         int y;
-        int height;
+        int height, width;
         int n = 0;
 
         if (vs->output.offset && !vs->audio_cap && !vs->force_update)
@@ -907,6 +907,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         job = vnc_job_new(vs);
 
         height = MIN(pixman_image_get_height(vd->server), vs->client_height);
+        width = MIN(pixman_image_get_width(vd->server), vs->client_width);
 
         y = 0;
         for (;;) {
@@ -925,8 +926,11 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
                                     VNC_DIRTY_BPL(vs), x);
             bitmap_clear(vs->dirty[y], x, x2 - x);
             h = find_and_clear_dirty_height(vs, y, x, x2, height);
-            n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
-                                  (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            x2 = MIN(x2, width / VNC_DIRTY_PIXELS_PER_BIT);
+            if (x2 > x) {
+                n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
+                                      (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
+            }
         }
 
         vnc_job_push(job);