diff --git a/core/decoders/tight.js b/core/decoders/tight.js index 1532a887..6fa06a0e 100644 --- a/core/decoders/tight.js +++ b/core/decoders/tight.js @@ -11,6 +11,7 @@ import * as Log from '../util/logging.js'; import Inflator from "../inflator.js"; +import { hashUInt8Array } from '../util/int.js'; export default class TightDecoder { constructor(display) { @@ -21,6 +22,8 @@ export default class TightDecoder { this._len = 0; this._enableQOI = false; this._displayGlobal = display; + this._lastTransparentRectHash = ''; + this._lastTransparentRectInfo = ''; this._zlibs = []; for (let i = 0; i < 4; i++) { @@ -183,44 +186,52 @@ export default class TightDecoder { return true; } - // intensity tinted _itRect(x, y, width, height, sock, display, depth, frame_id) { let data = this._readData(sock); if (data === null) { return false; } + + //filter out consecutive redundant data + let h = hashUInt8Array(data); + let info = `${x}.${y}.${width}.${height}` + if (!(h === this._lastTransparentRectHash && info === this._lastTransparentRectInfo)) { + const r = data[0]; + const g = data[1]; + const b = data[2]; + const a = data[3]; - const r = data[0]; - const g = data[1]; - const b = data[2]; - const a = data[3]; + const uncompressedSize = Math.floor(width * height / 2 + 1); - const uncompressedSize = width * height / 2 + 1; + this._itzlib.reset(); + this._itzlib.setInput(data.slice(4)); + data = this._itzlib.inflate(uncompressedSize); + this._itzlib.setInput(null); - this._itzlib.reset(); - this._itzlib.setInput(data.slice(4)); - data = this._itzlib.inflate(uncompressedSize); - this._itzlib.setInput(null); + // unpack + let rgba = new Uint8Array(width * height * 4 + 4); + for (let i = 0, d = 0; i < uncompressedSize; i++, d += 8) { + let p = data[i]; - // unpack - let rgba = new Uint8Array(width * height * 4 + 4); - for (let i = 0, d = 0; i < uncompressedSize; i++, d += 8) { - let p = data[i]; + rgba[d + 0] = r; + rgba[d + 1] = g; + rgba[d + 2] = b; + rgba[d + 3] = a * ((p & 15) << 4) / 255; - rgba[d + 0] = r; - rgba[d + 1] = g; - rgba[d + 2] = b; - rgba[d + 3] = a * ((p & 15) << 4) / 255; + rgba[d + 4] = r; + rgba[d + 5] = g; + rgba[d + 6] = b; + rgba[d + 7] = a * (p & 240) / 255; + } - rgba[d + 4] = r; - rgba[d + 5] = g; - rgba[d + 6] = b; - rgba[d + 7] = a * (p & 240) / 255; + let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height); + display.transparentRect(x, y, width, height, img, frame_id, h); + this._lastTransparentRectHash = h; + this._lastTransparentRectInfo = info; + } else { + display.dummyRect(x, y, width, height, frame_id); } - let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height); - display.transparentRect(x, y, width, height, img, frame_id); - return true; } diff --git a/core/display.js b/core/display.js index 984eec00..7da83173 100644 --- a/core/display.js +++ b/core/display.js @@ -32,6 +32,9 @@ export default class Display { this._maxAsyncFrameQueue = 3; this._clearAsyncQueue(); this._syncFrameQueue = []; + this._transparentOverlayImg = null; + this._transparentOverlayRect = null; + this._lastTransparentRectId = ""; this._flushing = false; @@ -75,7 +78,7 @@ export default class Display { if (delta > 0) { this._fps = (this._flipCnt / (delta / 1000)).toFixed(2); } - Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect); + Log.Debug('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect); this._flipCnt = 0; this._lastFlip = Date.now(); }.bind(this), 5000); @@ -107,6 +110,12 @@ export default class Display { channel: null }]; + //optional offscreen canvas + this._enableCanvasBuffer = false; + this._backbuffer = document.createElement('canvas'); + this._drawCtx = this._backbuffer.getContext('2d'); + this._damageBounds = { left: 0, top: 0, right: this._backbuffer.width, bottom: this._backbuffer.height }; + // ===== EVENT HANDLERS ===== this.onflush = () => { }; // A flush request has finished @@ -123,6 +132,29 @@ export default class Display { // ===== PROPERTIES ===== + get enableCanvasBuffer() { return this._enableCanvasBuffer; } + set enableCanvasBuffer(value) { + if (value === this._enableCanvasBuffer) { return; } + + this._enableCanvasBuffer = value; + + + if (value && this._target) + { + //copy current visible canvas to backbuffer + let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height); + this._drawCtx.putImageData(saveImg, 0, 0); + + if (this._transparentOverlayImg) { + this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true); + } + } else if (!value && this._target) { + //copy backbuffer to canvas to clear any overlays + let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height); + this._drawCtx.putImageData(saveImg, 0, 0); + } + } + get screens() { return this._screens; } get screenID() { return this._screenID; } get screenIndex() { @@ -419,14 +451,23 @@ export default class Display { } const vp = this._screens[0]; - if (vp.serverWidth !== width || vp.serverHeight !== height) { + const canvas = this._target; + if (canvas.width !== width || canvas.height !== height) { + let saveImg = null; + if (canvas.width > 0 && canvas.height > 0) { + saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height); + } + vp.serverWidth = width; vp.serverHeight = height; - const canvas = this._target; canvas.width = width; canvas.height = height; + if (saveImg) { + this._targetCtx.putImageData(saveImg, 0, 0); + } + // The position might need to be updated if we've grown this.viewportChangePos(0, 0); @@ -455,32 +496,36 @@ export default class Display { this._fbWidth = width; this._fbHeight = height; - const canvas = this._target; + let canvas = this._backbuffer; if (canvas == undefined) { return; } + if (this._screens.length > 0) { width = this._screens[0].serverWidth; height = this._screens[0].serverHeight; } - if (canvas.width !== width || canvas.height !== height) { + if (canvas.width !== width || canvas.height !== height) { // We have to save the canvas data since changing the size will clear it let saveImg = null; if (canvas.width > 0 && canvas.height > 0) { - saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height); + saveImg = this._drawCtx.getImageData(0, 0, canvas.width, canvas.height); } if (canvas.width !== width) { canvas.width = width; + } if (canvas.height !== height) { canvas.height = height; } if (saveImg) { - this._targetCtx.putImageData(saveImg, 0, 0); + this._drawCtx.putImageData(saveImg, 0, 0); } } + + // Readjust the viewport as it may be incorrectly sized // and positioned const vp = this._screens[0]; @@ -556,7 +601,11 @@ export default class Display { this._asyncRenderQPush(rect); } else { this._setFillColor(color); - this._targetCtx.fillRect(x, y, width, height); + if (this._enableCanvasBuffer) { + this._drawCtx.fillRect(x, y, width, height); + } else { + this._targetCtx.fillRect(x, y, width, height); + } } } @@ -575,6 +624,9 @@ export default class Display { this._processRectScreens(rect); this._asyncRenderQPush(rect); } else { + let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx); + let sourceCvs = ((this._enableCanvasBuffer) ? this._backbuffer : this._target); + // Due to this bug among others [1] we need to disable the image-smoothing to // avoid getting a blur effect when copying data. // @@ -582,12 +634,12 @@ export default class Display { // // We need to set these every time since all properties are reset // when the the size is changed - this._targetCtx.mozImageSmoothingEnabled = false; - this._targetCtx.webkitImageSmoothingEnabled = false; - this._targetCtx.msImageSmoothingEnabled = false; - this._targetCtx.imageSmoothingEnabled = false; + targetCtx.mozImageSmoothingEnabled = false; + targetCtx.webkitImageSmoothingEnabled = false; + targetCtx.msImageSmoothingEnabled = false; + targetCtx.imageSmoothingEnabled = false; - this._targetCtx.drawImage(this._target, + targetCtx.drawImage(sourceCvs, oldX, oldY, w, h, newX, newY, w, h); } @@ -625,7 +677,7 @@ export default class Display { this._asyncRenderQPush(rect); } - transparentRect(x, y, width, height, img, frame_id) { + transparentRect(x, y, width, height, img, frame_id, hashId) { /* The internal logic cannot handle empty images, so bail early */ if ((width === 0) || (height === 0)) { return; @@ -638,21 +690,35 @@ export default class Display { 'y': y, 'width': width, 'height': height, - 'frame_id': frame_id + 'frame_id': frame_id, + 'arr': img, + 'hash_id': hashId } this._processRectScreens(rect); if (rect.inPrimary) { let imageBmpPromise = createImageBitmap(img); - imageBmpPromise.then( function(img) { - rect.img = img; - rect.img.complete = true; - }.bind(rect) ); - } - if (rect.inSecondary) { - rect.arr = img; + imageBmpPromise.then( function(bitmap) { + this._transparentOverlayImg = bitmap; + this.enableCanvasBuffer = true; + }.bind(this) ); } + this._transparentOverlayRect = rect; + this._asyncRenderQPush(rect); + } + + dummyRect(x, y, width, height, frame_id) { + let rect = { + 'type': 'dummy', + 'img': null, + 'x': x, + 'y': y, + 'width': width, + 'height': height, + 'frame_id': frame_id + } + this._processRectScreens(rect); this._asyncRenderQPush(rect); } @@ -680,7 +746,13 @@ export default class Display { arr.byteOffset + offset, width * height * 4); let img = new ImageData(data, width, height); - this._targetCtx.putImageData(img, x, y); + if (this._enableCanvasBuffer) { + this._drawCtx.putImageData(img, x, y); + } else { + this._targetCtx.putImageData(img, x, y); + + } + } } @@ -698,16 +770,21 @@ export default class Display { this._processRectScreens(rect); this._asyncRenderQPush(rect); } else { - this._targetCtx.putImageData(arr, x, y); + if (this._enableCanvasBuffer) { + this._drawCtx.putImageData(arr, x, y); + } else { + this._targetCtx.putImageData(arr, x, y); + } } } - drawImage(img, x, y, w, h) { + drawImage(img, x, y, w, h, overlay=false) { try { + let targetCtx = ((this._enableCanvasBuffer && !overlay) ? this._drawCtx : this._targetCtx); if (img.width != w || img.height != h) { - this._targetCtx.drawImage(img, x, y, w, h); + targetCtx.drawImage(img, x, y, w, h); } else { - this._targetCtx.drawImage(img, x, y); + targetCtx.drawImage(img, x, y); } } catch (error) { Log.Error('Invalid image recieved.'); //KASM-2090 @@ -737,9 +814,13 @@ export default class Display { // ===== PRIVATE METHODS ===== + _writeCtxBuffer() { + //TODO: KASM-5450 Damage tracking with transparent rect overlay support + this._targetCtx.drawImage(this._backbuffer, 0, 0); + } + _handleSecondaryDisplayMessage(event) { if (!this._isPrimaryDisplay && event.data) { - switch (event.data.eventType) { case 'rect': let rect = event.data.rect; @@ -755,9 +836,11 @@ export default class Display { break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); - imageBmpPromise.then(function(rect, img) { - rect.img.complete = true; - }).bind(this, rect); + imageBmpPromise.then( function(img) { + this._transparentOverlayImg = img; + this.enableCanvasBuffer = true; + }.bind(this) ); + this._transparentOverlayRect = rect; break; } this._syncFrameQueue.push(rect); @@ -768,7 +851,10 @@ export default class Display { case 'registered': if (!this._isPrimaryDisplay) { this._screens[0].screenIndex = event.data.screenIndex; - Log.Error(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`); + Log.Info(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`); + if (this._screens.length > 0) { + this.resize(this._screens[0].serverWidth, this._screens[0].serverHeight); + } } break; } @@ -776,6 +862,7 @@ export default class Display { } _pushSyncRects() { + let drawRectCnt = 0; whileLoop: while (this._syncFrameQueue.length > 0) { const a = this._syncFrameQueue[0]; @@ -805,24 +892,21 @@ export default class Display { } } break; - case 'transparent': - if (a.img.complete) { - this.drawImage(a.img, pos.x, pos.y, a.width, a.height); - } else { - if (this._syncFrameQueue.length > 1000) { - this._syncFrameQueue.shift(); - this._droppedRects++; - } else { - break whileLoop; - } - } - break; default: - Log.Warn(`Unknown rect type: ${rect}`); + this._syncFrameQueue.shift(); + continue; } + drawRectCnt++; this._syncFrameQueue.shift(); } + if (this._enableCanvasBuffer && drawRectCnt > 0) { + this._writeCtxBuffer(); + if (this._transparentOverlayImg) { + this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true); + } + } + if (this._syncFrameQueue.length > 0) { window.requestAnimationFrame( () => { this._pushSyncRects(); }); } @@ -963,8 +1047,6 @@ export default class Display { this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.addEventListener('load', () => { this._asyncFrameComplete(frameIx); }); this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; return; - } else if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'transparent' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img) { - return; } currentFrameRectIx++; @@ -991,8 +1073,8 @@ export default class Display { this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ]); } - let transparent_rects = []; let secondaryScreenRects = 0; + let primaryScreenRects = 0; //render the selected frame for (let i = 0; i < frame.length; i++) { @@ -1018,38 +1100,41 @@ export default class Display { case 'img': this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height); break; - case 'transparent': - transparent_rects.push(a); - break; + default: + continue; } + primaryScreenRects++; } else { - if (a.img) { - a.img = null; - } - - if (a.type !== 'flip') { - secondaryScreenRects++; - this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + switch (a.type) { + case 'dummy': + case 'transparent': + case 'flip': + break; + default: + secondaryScreenRects++; + a.img = null; + this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); } } } } - //rects with transparency get applied last - for (let i = 0; i < transparent_rects.length; i++) { - const a = transparent_rects[i]; - let screenIndexes = this._getRectScreenIndexes(a); + if (this._enableCanvasBuffer) { - for (let sI = 0; sI < screenLocations.length; sI++) { - let screenLocation = a.screenLocations[sI]; - if (sI == 0) { - if (a.img) { - this.drawImage(a.img, a.x, a.y, a.width, a.height); - } - } else { - secondaryScreenRects++; - this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + if (primaryScreenRects > 0) { + this._writeCtxBuffer(); + } + + if (this._transparentOverlayImg) { + if (primaryScreenRects > 0) { + this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true); } + if (secondaryScreenRects > 0 && this._lastTransparentRectId !== this._transparentOverlayRect.hash_id) { + for (let sI = 1; sI < this._transparentOverlayRect.screenLocations.length; sI++) { + this._screens[this._transparentOverlayRect.screenLocations[sI].screenIndex].channel.postMessage({ eventType: 'rect', rect: this._transparentOverlayRect, screenLocationIndex: sI }); + } + } + this._lastTransparentRectId = this._transparentOverlayRect.hash_id; } } @@ -1065,6 +1150,11 @@ export default class Display { this._flushing = false; this.onflush(); } + + // if there is more data in queue, then keep checking + if (this._asyncFrameQueue[0][2].length > 0) { + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + } } else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] == this._asyncFrameQueue[0][2].length) { //how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn this._asyncFrameQueue[0][5] += 1; @@ -1141,8 +1231,9 @@ export default class Display { _setFillColor(color) { const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')'; + let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx); if (newStyle !== this._prevDrawStyle) { - this._targetCtx.fillStyle = newStyle; + targetCtx.fillStyle = newStyle; this._prevDrawStyle = newStyle; } } diff --git a/core/inflator.js b/core/inflator.js index 4b337607..e2844c6f 100644 --- a/core/inflator.js +++ b/core/inflator.js @@ -54,7 +54,7 @@ export default class Inflate { } if (this.strm.next_out != expected) { - throw new Error("Incomplete zlib block"); + throw new Error("Incomplete zlib block, got " + this.strm.next_out + " expected " + expected); } return new Uint8Array(this.strm.output.buffer, 0, this.strm.next_out); diff --git a/core/rfb.js b/core/rfb.js index 44b98bc5..27c35c01 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -217,7 +217,6 @@ export default class RFB extends EventTargetMixin { this._gestureLastMagnitudeY = 0; // Secondary Displays - this._secondaryDisplays = {}; this._supportsBroadcastChannel = (typeof BroadcastChannel !== "undefined"); if (this._supportsBroadcastChannel) { this._controlChannel = new BroadcastChannel(this._connectionID); @@ -1750,8 +1749,6 @@ export default class RFB extends EventTargetMixin { Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); break; case 'reattach': - console.log('reattach message') - console.log(event.data) this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); size = this._screenSize(); RFB.messages.setDesktopSize(this._sock, size, this._screenFlags);