Bugfix/kasm 5417 watermark multi mon (#85)

* KASM-5417 fix bug watermrk multi-display
* Force the zlib expected size to an int, it was unexpectedly float
* KASM-5427 transparent rects not drawn on primary display
* add double buffer when transparent rects are used
* refactor of transparent rects
* Use a more descriptive zlib error
* account for dropped transparent rects
* fix bug with multi-monitor with overlay

core/decoders/tight.js

@@ -11,6 +11,7 @@
 
 import * as Log from '../util/logging.js';
 import Inflator from "../inflator.js";
+import { hashUInt8Array } from '../util/int.js';
 
 export default class TightDecoder {
     constructor(display) {
@@ -21,6 +22,8 @@ export default class TightDecoder {
         this._len = 0;
         this._enableQOI = false;
         this._displayGlobal = display;
+        this._lastTransparentRectHash = '';
+        this._lastTransparentRectInfo = '';
 
         this._zlibs = [];
         for (let i = 0; i < 4; i++) {
@@ -183,19 +186,22 @@ export default class TightDecoder {
         return true;
     }
 
-    // intensity tinted
     _itRect(x, y, width, height, sock, display, depth, frame_id) {
         let data = this._readData(sock);
         if (data === null) {
             return false;
         }
         
+        //filter out consecutive redundant data
+        let h = hashUInt8Array(data);
+        let info = `${x}.${y}.${width}.${height}`
+        if (!(h === this._lastTransparentRectHash && info === this._lastTransparentRectInfo)) {
             const r = data[0];
             const g = data[1];
             const b = data[2];
             const a = data[3];
 
-        const uncompressedSize = width * height / 2 + 1;
+            const uncompressedSize = Math.floor(width * height / 2 + 1);
 
             this._itzlib.reset();
             this._itzlib.setInput(data.slice(4));
@@ -219,7 +225,12 @@ export default class TightDecoder {
             }
 
             let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height);
-        display.transparentRect(x, y, width, height, img, frame_id);
+            display.transparentRect(x, y, width, height, img, frame_id, h);
+            this._lastTransparentRectHash = h;
+            this._lastTransparentRectInfo = info;
+        } else {
+            display.dummyRect(x, y, width, height, frame_id);
+        }
 
         return true;
     }

core/display.js

@@ -32,6 +32,9 @@ export default class Display {
         this._maxAsyncFrameQueue = 3;
         this._clearAsyncQueue();
         this._syncFrameQueue = [];
+        this._transparentOverlayImg = null;
+        this._transparentOverlayRect = null;
+        this._lastTransparentRectId = "";
 
         this._flushing = false;
 
@@ -75,7 +78,7 @@ export default class Display {
             if (delta > 0) {
                 this._fps = (this._flipCnt / (delta / 1000)).toFixed(2);
             }
-            Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect);
+            Log.Debug('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect);
             this._flipCnt = 0;
             this._lastFlip = Date.now();
         }.bind(this), 5000);
@@ -107,6 +110,12 @@ export default class Display {
             channel: null
         }];
 
+        //optional offscreen canvas
+        this._enableCanvasBuffer = false;
+        this._backbuffer = document.createElement('canvas');
+        this._drawCtx = this._backbuffer.getContext('2d');
+        this._damageBounds = { left: 0, top: 0, right: this._backbuffer.width, bottom: this._backbuffer.height };
+
         // ===== EVENT HANDLERS =====
 
         this.onflush = () => {  }; // A flush request has finished
@@ -123,6 +132,29 @@ export default class Display {
 
     // ===== PROPERTIES =====
 
+    get enableCanvasBuffer() { return this._enableCanvasBuffer; }
+    set enableCanvasBuffer(value) {
+        if (value === this._enableCanvasBuffer) { return; }
+
+        this._enableCanvasBuffer = value;
+
+        
+        if (value && this._target)
+        {
+            //copy current visible canvas to backbuffer
+            let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height);
+            this._drawCtx.putImageData(saveImg, 0, 0);
+
+            if (this._transparentOverlayImg) {
+                this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
+            }
+        } else if (!value && this._target) {
+            //copy backbuffer to canvas to clear any overlays
+            let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height);
+            this._drawCtx.putImageData(saveImg, 0, 0);
+        }
+    }
+
     get screens() { return this._screens; }
     get screenID() { return this._screenID; }
     get screenIndex() {
@@ -419,14 +451,23 @@ export default class Display {
         }
 
         const vp = this._screens[0];
-        if (vp.serverWidth !== width || vp.serverHeight !== height) {
+        const canvas = this._target;
+        if (canvas.width !== width || canvas.height !== height) {
+            let saveImg = null;
+            if (canvas.width > 0 && canvas.height > 0) {
+                saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height);
+            }
+
             vp.serverWidth = width;
             vp.serverHeight = height;
 
-            const canvas = this._target;
             canvas.width = width;
             canvas.height = height;
 
+            if (saveImg) {
+                this._targetCtx.putImageData(saveImg, 0, 0);
+            }
+
             // The position might need to be updated if we've grown
             this.viewportChangePos(0, 0);
 
@@ -455,32 +496,36 @@ export default class Display {
         this._fbWidth = width;
         this._fbHeight = height;
 
-        const canvas = this._target;
+        let canvas = this._backbuffer;
         if (canvas == undefined) { return; }
+        
         if (this._screens.length > 0) {
             width = this._screens[0].serverWidth;
             height = this._screens[0].serverHeight;
         }
-        if (canvas.width !== width || canvas.height !== height) {
 
+        if (canvas.width !== width || canvas.height !== height) {
             // We have to save the canvas data since changing the size will clear it
             let saveImg = null;
             if (canvas.width > 0 && canvas.height > 0) {
-                saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height);
+                saveImg = this._drawCtx.getImageData(0, 0, canvas.width, canvas.height);
             }
 
             if (canvas.width !== width) {
                 canvas.width = width;
+
             }
             if (canvas.height !== height) {
                 canvas.height = height;
             }
 
             if (saveImg) {
-                this._targetCtx.putImageData(saveImg, 0, 0);
+                this._drawCtx.putImageData(saveImg, 0, 0);
             }
         }
 
+        
+
         // Readjust the viewport as it may be incorrectly sized
         // and positioned
         const vp = this._screens[0];
@@ -556,9 +601,13 @@ export default class Display {
             this._asyncRenderQPush(rect);
         } else {
             this._setFillColor(color);
+            if (this._enableCanvasBuffer) {
+                this._drawCtx.fillRect(x, y, width, height);
+            } else {
                 this._targetCtx.fillRect(x, y, width, height);
             }
         }
+    }
 
     copyImage(oldX, oldY, newX, newY, w, h, frame_id, fromQueue) {
         if (!fromQueue) {
@@ -575,6 +624,9 @@ export default class Display {
             this._processRectScreens(rect);
             this._asyncRenderQPush(rect);
         } else {
+            let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx);
+            let sourceCvs = ((this._enableCanvasBuffer) ? this._backbuffer : this._target);
+
             // Due to this bug among others [1] we need to disable the image-smoothing to
             // avoid getting a blur effect when copying data.
             //
@@ -582,12 +634,12 @@ export default class Display {
             //
             // We need to set these every time since all properties are reset
             // when the the size is changed
-            this._targetCtx.mozImageSmoothingEnabled = false;
+            targetCtx.mozImageSmoothingEnabled = false;
-            this._targetCtx.webkitImageSmoothingEnabled = false;
+            targetCtx.webkitImageSmoothingEnabled = false;
-            this._targetCtx.msImageSmoothingEnabled = false;
+            targetCtx.msImageSmoothingEnabled = false;
-            this._targetCtx.imageSmoothingEnabled = false;
+            targetCtx.imageSmoothingEnabled = false;
 
-            this._targetCtx.drawImage(this._target,
+            targetCtx.drawImage(sourceCvs,
                                     oldX, oldY, w, h,
                                     newX, newY, w, h);
         }
@@ -625,7 +677,7 @@ export default class Display {
         this._asyncRenderQPush(rect);
     }
 
-    transparentRect(x, y, width, height, img, frame_id) {
+    transparentRect(x, y, width, height, img, frame_id, hashId) {
         /* The internal logic cannot handle empty images, so bail early */
         if ((width === 0) || (height === 0)) {
             return;
@@ -638,21 +690,35 @@ export default class Display {
             'y': y,
             'width': width,
             'height': height,
-            'frame_id': frame_id
+            'frame_id': frame_id,
+            'arr': img,
+            'hash_id': hashId
         }
         this._processRectScreens(rect);
 
         if (rect.inPrimary) {
             let imageBmpPromise = createImageBitmap(img);
-            imageBmpPromise.then( function(img) {
+            imageBmpPromise.then( function(bitmap) {
-                rect.img = img;
+                this._transparentOverlayImg = bitmap;
-                rect.img.complete = true;
+                this.enableCanvasBuffer = true;
-            }.bind(rect) );
+            }.bind(this) );
-        } 
-        if (rect.inSecondary) {
-            rect.arr = img;
         }
 
+        this._transparentOverlayRect = rect;
+        this._asyncRenderQPush(rect);
+    }
+
+    dummyRect(x, y, width, height, frame_id) {
+        let rect = {
+            'type': 'dummy',
+            'img': null,
+            'x': x,
+            'y': y,
+            'width': width,
+            'height': height,
+            'frame_id': frame_id
+        }
+        this._processRectScreens(rect);
         this._asyncRenderQPush(rect);
     }
 
@@ -680,7 +746,13 @@ export default class Display {
                                              arr.byteOffset + offset,
                                              width * height * 4);
             let img = new ImageData(data, width, height);
+            if (this._enableCanvasBuffer) {
+                this._drawCtx.putImageData(img, x, y);
+            } else {
                 this._targetCtx.putImageData(img, x, y);
+                
+            }
+            
         }
     }
 
@@ -697,17 +769,22 @@ export default class Display {
             }
             this._processRectScreens(rect);
             this._asyncRenderQPush(rect);
+        } else {
+            if (this._enableCanvasBuffer) {
+                this._drawCtx.putImageData(arr, x, y);
             } else {
                 this._targetCtx.putImageData(arr, x, y);
             }
         }
+    }
 
-    drawImage(img, x, y, w, h) {
+    drawImage(img, x, y, w, h, overlay=false) {
         try {
+            let targetCtx = ((this._enableCanvasBuffer && !overlay) ? this._drawCtx : this._targetCtx);
             if (img.width != w || img.height != h) {
-                this._targetCtx.drawImage(img, x, y, w, h);
+                targetCtx.drawImage(img, x, y, w, h);
             } else {
-                this._targetCtx.drawImage(img, x, y);
+                targetCtx.drawImage(img, x, y);
             }
         } catch (error) {
             Log.Error('Invalid image recieved.'); //KASM-2090
@@ -737,9 +814,13 @@ export default class Display {
 
     // ===== PRIVATE METHODS =====
 
+    _writeCtxBuffer() {
+    	//TODO: KASM-5450 Damage tracking with transparent rect overlay support
+        this._targetCtx.drawImage(this._backbuffer, 0, 0);
+    }
+
     _handleSecondaryDisplayMessage(event) {
         if (!this._isPrimaryDisplay && event.data) {
-            
             switch (event.data.eventType) {
                 case 'rect':
                     let rect = event.data.rect;
@@ -755,9 +836,11 @@ export default class Display {
                             break;
                         case 'transparent':
                             let imageBmpPromise = createImageBitmap(rect.arr);
-                            imageBmpPromise.then(function(rect, img) {
+                            imageBmpPromise.then( function(img) {
-                                rect.img.complete = true;
+                                this._transparentOverlayImg = img;
-                            }).bind(this, rect);
+                                this.enableCanvasBuffer = true;
+                            }.bind(this) );
+                            this._transparentOverlayRect = rect;
                             break;
                     }
                     this._syncFrameQueue.push(rect);
@@ -768,7 +851,10 @@ export default class Display {
                 case 'registered':
                         if (!this._isPrimaryDisplay) {
                             this._screens[0].screenIndex = event.data.screenIndex;
-                            Log.Error(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`);
+                            Log.Info(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`);
+                            if (this._screens.length > 0) {
+                                this.resize(this._screens[0].serverWidth, this._screens[0].serverHeight);
+                            }
                         }
                     break;
             }
@@ -776,6 +862,7 @@ export default class Display {
     }
 
     _pushSyncRects() {
+        let drawRectCnt = 0;
         whileLoop:
         while (this._syncFrameQueue.length > 0) {
             const a = this._syncFrameQueue[0];
@@ -805,22 +892,19 @@ export default class Display {
                         }
                     }
                     break;
-                case 'transparent':
-                    if (a.img.complete) {
-                        this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
-                    } else {
-                        if (this._syncFrameQueue.length > 1000) {
-                            this._syncFrameQueue.shift();
-                            this._droppedRects++;
-                        } else {
-                            break whileLoop;
-                        }
-                    }
-                    break;
                 default:
-                    Log.Warn(`Unknown rect type: ${rect}`);
-            }
                     this._syncFrameQueue.shift();
+                    continue;
+            }
+            drawRectCnt++;
+            this._syncFrameQueue.shift();
+        }
+
+        if (this._enableCanvasBuffer && drawRectCnt > 0) {
+            this._writeCtxBuffer();
+            if (this._transparentOverlayImg) {
+                this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
+            }
         }
 
         if (this._syncFrameQueue.length > 0) {
@@ -963,8 +1047,6 @@ export default class Display {
                     this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.addEventListener('load', () => { this._asyncFrameComplete(frameIx); });
                     this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
                     return;
-                } else if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'transparent' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img) {
-                    return;
                 }
 
                 currentFrameRectIx++;
@@ -991,8 +1073,8 @@ export default class Display {
                 this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ]);
             }
 
-            let transparent_rects = [];
             let secondaryScreenRects = 0;
+            let primaryScreenRects = 0;
             
             //render the selected frame
             for (let i = 0; i < frame.length; i++) {
@@ -1018,38 +1100,41 @@ export default class Display {
                             case 'img':
                                 this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height);
                                 break;
+                            default:
+                                continue;
+                        }
+                        primaryScreenRects++;
+                    } else {
+                        switch (a.type) {
+                            case 'dummy':
                             case 'transparent':
-                                transparent_rects.push(a);
+                            case 'flip':
                                 break;
-                        }
+                            default:
-                    } else {
+                                secondaryScreenRects++;
-                        if (a.img) {
                                 a.img = null;
-                        }
-
-                        if (a.type !== 'flip') {
-                            secondaryScreenRects++;
                                 this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });
                         }
                     }
                 }
             }
 
-            //rects with transparency get applied last
+            if (this._enableCanvasBuffer) {
-            for (let i = 0; i < transparent_rects.length; i++) {
-                const a = transparent_rects[i];
-                let screenIndexes = this._getRectScreenIndexes(a);
 
-                for (let sI = 0; sI < screenLocations.length; sI++) {
+                if (primaryScreenRects > 0) {
-                    let screenLocation = a.screenLocations[sI];
+                    this._writeCtxBuffer();
-                    if (sI == 0) {
-                        if (a.img) {
-                            this.drawImage(a.img, a.x, a.y, a.width, a.height);
                 }
-                    } else {
+                
-                        secondaryScreenRects++;
+                if (this._transparentOverlayImg) { 
-                        this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });
+                    if (primaryScreenRects > 0) {
+                        this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
                     }
+                    if (secondaryScreenRects > 0 && this._lastTransparentRectId !== this._transparentOverlayRect.hash_id) {
+                        for (let sI = 1; sI < this._transparentOverlayRect.screenLocations.length; sI++) {
+                            this._screens[this._transparentOverlayRect.screenLocations[sI].screenIndex].channel.postMessage({ eventType: 'rect', rect: this._transparentOverlayRect, screenLocationIndex: sI });
+                        }
+                    }
+                    this._lastTransparentRectId = this._transparentOverlayRect.hash_id;
                 }
             }
 
@@ -1065,6 +1150,11 @@ export default class Display {
                 this._flushing = false;
                 this.onflush();
             }
+
+            // if there is more data in queue, then keep checking
+            if (this._asyncFrameQueue[0][2].length > 0) {
+                window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
+            }
         } else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] == this._asyncFrameQueue[0][2].length) {
             //how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn
             this._asyncFrameQueue[0][5] += 1;
@@ -1141,8 +1231,9 @@ export default class Display {
 
     _setFillColor(color) {
         const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')';
+        let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx);
         if (newStyle !== this._prevDrawStyle) {
-            this._targetCtx.fillStyle = newStyle;
+            targetCtx.fillStyle = newStyle;
             this._prevDrawStyle = newStyle;
         }
     }

core/inflator.js

@@ -54,7 +54,7 @@ export default class Inflate {
         }
 
         if (this.strm.next_out != expected) {
-            throw new Error("Incomplete zlib block");
+            throw new Error("Incomplete zlib block, got " + this.strm.next_out + " expected " + expected);
         }
 
         return new Uint8Array(this.strm.output.buffer, 0, this.strm.next_out);

core/rfb.js

@@ -217,7 +217,6 @@ export default class RFB extends EventTargetMixin {
         this._gestureLastMagnitudeY = 0;
 
         // Secondary Displays
-        this._secondaryDisplays = {};
         this._supportsBroadcastChannel = (typeof BroadcastChannel !== "undefined");
         if (this._supportsBroadcastChannel) {
             this._controlChannel = new BroadcastChannel(this._connectionID);
@@ -1750,8 +1749,6 @@ export default class RFB extends EventTargetMixin {
                     Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`);
                     break;
                 case 'reattach':
-                    console.log('reattach message')
-                    console.log(event.data)
                     this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth);
                     size = this._screenSize();
                     RFB.messages.setDesktopSize(this._sock, size, this._screenFlags);