Add support for VMware cursor encoding

Supports both classic cursor type and alpha cursor type. In classic
mode the server can send 'inverted' pixels for the cursor, our code
does not support this but handles these pixels as opaque black.

Co-authored-by: Samuel Mannehed <samuel@cendio.se>

diff --git a/core/encodings.js b/core/encodings.js
index 9fd38d58fcc332e341eb70dbd58bbc77b87e1d42..2b3486729709c9d65164015b81e446de1b4a9fff 100644 (file)
--- a/core/encodings.js
+++ b/core/encodings.js
@@ -26,6 +26,7 @@ export const encodings = {
     pseudoEncodingContinuousUpdates: -313,
     pseudoEncodingCompressLevel9: -247,
     pseudoEncodingCompressLevel0: -256,
+    pseudoEncodingVMwareCursor: 0x574d5664
 };
 
 export function encodingName(num) {

diff --git a/core/rfb.js b/core/rfb.js
index 1761500a0fc4ccf6343f72eb6032d3552787c429..59bd8391749cdd6be8f367f1632d8c0558b0f627 100644 (file)
--- a/core/rfb.js
+++ b/core/rfb.js
@@ -1239,6 +1239,7 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.pseudoEncodingContinuousUpdates);
 
         if (this._fb_depth == 24) {
+            encs.push(encodings.pseudoEncodingVMwareCursor);
             encs.push(encodings.pseudoEncodingCursor);
         }
 
@@ -1488,6 +1489,9 @@ export default class RFB extends EventTargetMixin {
                 this._FBU.rects = 1; // Will be decreased when we return
                 return true;
 
+            case encodings.pseudoEncodingVMwareCursor:
+                return this._handleVMwareCursor();
+
             case encodings.pseudoEncodingCursor:
                 return this._handleCursor();
 
@@ -1515,6 +1519,122 @@ export default class RFB extends EventTargetMixin {
         }
     }
 
+    _handleVMwareCursor() {
+        const hotx = this._FBU.x;  // hotspot-x
+        const hoty = this._FBU.y;  // hotspot-y
+        const w = this._FBU.width;
+        const h = this._FBU.height;
+        if (this._sock.rQwait("VMware cursor encoding", 1)) {
+            return false;
+        }
+
+        const cursor_type = this._sock.rQshift8();
+
+        this._sock.rQshift8(); //Padding
+
+        let rgba;
+        const bytesPerPixel = 4;
+
+        //Classic cursor
+        if (cursor_type == 0) {
+            //Used to filter away unimportant bits.
+            //OR is used for correct conversion in js.
+            const PIXEL_MASK = 0xffffff00 | 0;
+            rgba = new Array(w * h * bytesPerPixel);
+
+            if (this._sock.rQwait("VMware cursor classic encoding",
+                                  (w * h * bytesPerPixel) * 2, 2)) {
+                return false;
+            }
+
+            let and_mask = new Array(w * h);
+            for (let pixel = 0; pixel < (w * h); pixel++) {
+                and_mask[pixel] = this._sock.rQshift32();
+            }
+
+            let xor_mask = new Array(w * h);
+            for (let pixel = 0; pixel < (w * h); pixel++) {
+                xor_mask[pixel] = this._sock.rQshift32();
+            }
+
+            for (let pixel = 0; pixel < (w * h); pixel++) {
+                if (and_mask[pixel] == 0) {
+                    //Fully opaque pixel
+                    let bgr = xor_mask[pixel];
+                    let r   = bgr >> 8  & 0xff;
+                    let g   = bgr >> 16 & 0xff;
+                    let b   = bgr >> 24 & 0xff;
+
+                    rgba[(pixel * bytesPerPixel)     ] = r;    //r
+                    rgba[(pixel * bytesPerPixel) + 1 ] = g;    //g
+                    rgba[(pixel * bytesPerPixel) + 2 ] = b;    //b
+                    rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; //a
+
+                } else if ((and_mask[pixel] & PIXEL_MASK) ==
+                           PIXEL_MASK) {
+                    //Only screen value matters, no mouse colouring
+                    if (xor_mask[pixel] == 0) {
+                        //Transparent pixel
+                        rgba[(pixel * bytesPerPixel)     ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 3 ] = 0x00;
+
+                    } else if ((xor_mask[pixel] & PIXEL_MASK) ==
+                               PIXEL_MASK) {
+                        //Inverted pixel, not supported in browsers.
+                        //Fully opaque instead.
+                        rgba[(pixel * bytesPerPixel)     ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
+
+                    } else {
+                        //Unhandled xor_mask
+                        rgba[(pixel * bytesPerPixel)     ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
+                        rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
+                    }
+
+                } else {
+                    //Unhandled and_mask
+                    rgba[(pixel * bytesPerPixel)     ] = 0x00;
+                    rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
+                    rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
+                    rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
+                }
+            }
+
+        //Alpha cursor.
+        } else if (cursor_type == 1) {
+            if (this._sock.rQwait("VMware cursor alpha encoding",
+                                  (w * h * 4), 2)) {
+                return false;
+            }
+
+            rgba = new Array(w * h * bytesPerPixel);
+
+            for (let pixel = 0; pixel < (w * h); pixel++) {
+                let data = this._sock.rQshift32();
+
+                rgba[(pixel * 4)     ] = data >> 8 & 0xff;  //r
+                rgba[(pixel * 4) + 1 ] = data >> 16 & 0xff; //g
+                rgba[(pixel * 4) + 2 ] = data >> 24 & 0xff; //b
+                rgba[(pixel * 4) + 3 ] = data & 0xff;       //a
+            }
+
+        } else {
+            Log.Warn("The given cursor type is not supported: "
+                      + cursor_type + " given.");
+            return false;
+        }
+
+        this._updateCursor(rgba, hotx, hoty, w, h);
+
+        return true;
+    }
+
     _handleCursor() {
         const hotx = this._FBU.x;  // hotspot-x
         const hoty = this._FBU.y;  // hotspot-y

diff --git a/tests/test.rfb.js b/tests/test.rfb.js
index 1563a1e24f9d3da1ee37193714fc75e5beef2647..c0db906b76bfa6b460578809fb3c9f9578f01f3c 100644 (file)
--- a/tests/test.rfb.js
+++ b/tests/test.rfb.js
@@ -2124,6 +2124,170 @@ describe('Remote Frame Buffer Protocol Client', function () {
                     });
                 });
 
+                describe('the VMware Cursor pseudo-encoding handler', function () {
+                    beforeEach(function () {
+                        sinon.spy(client._cursor, 'change');
+                    });
+                    afterEach(function () {
+                        client._cursor.change.resetHistory();
+                    });
+
+                    it('should handle the VMware cursor pseudo-encoding', function () {
+                        let data = [0x00, 0x00, 0xff, 0,
+                                    0x00, 0xff, 0x00, 0,
+                                    0x00, 0xff, 0x00, 0,
+                                    0x00, 0x00, 0xff, 0];
+                        let rect = [];
+                        push8(rect, 0);
+                        push8(rect, 0);
+
+                        //AND-mask
+                        for (let i = 0; i < data.length; i++) {
+                            push8(rect, data[i]);
+                        }
+                        //XOR-mask
+                        for (let i = 0; i < data.length; i++) {
+                            push8(rect, data[i]);
+                        }
+
+                        send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
+                                        encoding: 0x574d5664}],
+                                     [rect], client);
+                        expect(client._FBU.rects).to.equal(0);
+                    });
+
+                    it('should handle insufficient cursor pixel data', function () {
+
+                        // Specified 14x23 pixels for the cursor,
+                        // but only send 2x2 pixels worth of data
+                        let w = 14;
+                        let h = 23;
+                        let data = [0x00, 0x00, 0xff, 0,
+                                    0x00, 0xff, 0x00, 0];
+                        let rect = [];
+
+                        push8(rect, 0);
+                        push8(rect, 0);
+
+                        //AND-mask
+                        for (let i = 0; i < data.length; i++) {
+                            push8(rect, data[i]);
+                        }
+                        //XOR-mask
+                        for (let i = 0; i < data.length; i++) {
+                            push8(rect, data[i]);
+                        }
+
+                        send_fbu_msg([{ x: 0, y: 0, width: w, height: h,
+                                        encoding: 0x574d5664}],
+                                     [rect], client);
+
+                        // expect one FBU to remain unhandled
+                        expect(client._FBU.rects).to.equal(1);
+                    });
+
+                    it('should update the cursor when type is classic', function () {
+                        let and_mask =
+                            [0xff, 0xff, 0xff, 0xff,  //Transparent
+                             0xff, 0xff, 0xff, 0xff,  //Transparent
+                             0x00, 0x00, 0x00, 0x00,  //Opaque
+                             0xff, 0xff, 0xff, 0xff]; //Inverted
+
+                        let xor_mask =
+                            [0x00, 0x00, 0x00, 0x00,  //Transparent
+                             0x00, 0x00, 0x00, 0x00,  //Transparent
+                             0x11, 0x22, 0x33, 0x44,  //Opaque
+                             0xff, 0xff, 0xff, 0x44]; //Inverted
+
+                        let rect = [];
+                        push8(rect, 0); //cursor_type
+                        push8(rect, 0); //padding
+                        let hotx = 0;
+                        let hoty = 0;
+                        let w = 2;
+                        let h = 2;
+
+                        //AND-mask
+                        for (let i = 0; i < and_mask.length; i++) {
+                            push8(rect, and_mask[i]);
+                        }
+                        //XOR-mask
+                        for (let i = 0; i < xor_mask.length; i++) {
+                            push8(rect, xor_mask[i]);
+                        }
+
+                        let expected_rgba = [0x00, 0x00, 0x00, 0x00,
+                                             0x00, 0x00, 0x00, 0x00,
+                                             0x33, 0x22, 0x11, 0xff,
+                                             0x00, 0x00, 0x00, 0xff];
+
+                        send_fbu_msg([{ x: hotx, y: hoty,
+                                        width: w, height: h,
+                                        encoding: 0x574d5664}],
+                                     [rect], client);
+
+                        expect(client._cursor.change)
+                            .to.have.been.calledOnce;
+                        expect(client._cursor.change)
+                            .to.have.been.calledWith(expected_rgba,
+                                                     hotx, hoty,
+                                                     w, h);
+                    });
+
+                    it('should update the cursor when type is alpha', function () {
+                        let data = [0xee, 0x55, 0xff, 0x00, // bgra
+                                    0x00, 0xff, 0x00, 0xff,
+                                    0x00, 0xff, 0x00, 0x22,
+                                    0x00, 0xff, 0x00, 0x22,
+                                    0x00, 0xff, 0x00, 0x22,
+                                    0x00, 0x00, 0xff, 0xee];
+                        let rect = [];
+                        push8(rect, 1); //cursor_type
+                        push8(rect, 0); //padding
+                        let hotx = 0;
+                        let hoty = 0;
+                        let w = 3;
+                        let h = 2;
+
+                        for (let i = 0; i < data.length; i++) {
+                            push8(rect, data[i]);
+                        }
+
+                        let expected_rgba = [0xff, 0x55, 0xee, 0x00,
+                                             0x00, 0xff, 0x00, 0xff,
+                                             0x00, 0xff, 0x00, 0x22,
+                                             0x00, 0xff, 0x00, 0x22,
+                                             0x00, 0xff, 0x00, 0x22,
+                                             0xff, 0x00, 0x00, 0xee];
+
+                        send_fbu_msg([{ x: hotx, y: hoty,
+                                        width: w, height: h,
+                                        encoding: 0x574d5664}],
+                                     [rect], client);
+
+                        expect(client._cursor.change)
+                            .to.have.been.calledOnce;
+                        expect(client._cursor.change)
+                            .to.have.been.calledWith(expected_rgba,
+                                                     hotx, hoty,
+                                                     w, h);
+                    });
+
+                    it('should not update cursor when incorrect cursor type given', function () {
+                        let rect = [];
+                        push8(rect, 3); // invalid cursor type
+                        push8(rect, 0); // padding
+
+                        client._cursor.change.resetHistory();
+                        send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
+                                        encoding: 0x574d5664}],
+                                     [rect], client);
+
+                        expect(client._cursor.change)
+                            .to.not.have.been.called;
+                    });
+                });
+
                 it('should handle the last_rect pseudo-encoding', function () {
                     send_fbu_msg([{ x: 0, y: 0, width: 0, height: 0, encoding: -224}], [[]], client, 100);
                     expect(client._FBU.rects).to.equal(0);