diff --git a/app/ui.js b/app/ui.js index f43a9355..04a5c825 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1,5 +1,6 @@ /* - * noVNC: HTML5 VNC client + * KasmVNC: HTML5 VNC client + * Copyright (C) 2020 Kasm Technologies * Copyright (C) 2019 The noVNC Authors * Licensed under MPL 2.0 (see LICENSE.txt) * diff --git a/core/decoders/tight.js b/core/decoders/tight.js index fea7bca5..47b2d486 100644 --- a/core/decoders/tight.js +++ b/core/decoders/tight.js @@ -1,5 +1,6 @@ /* - * noVNC: HTML5 VNC client + * KasmVNC: HTML5 VNC client + * Copyright (C) 2020 Kasm Technologies * Copyright (C) 2019 The noVNC Authors * (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca) * Licensed under MPL 2.0 (see LICENSE.txt) diff --git a/core/decoders/udp.js b/core/decoders/udp.js index f59e8119..f40e1c40 100644 --- a/core/decoders/udp.js +++ b/core/decoders/udp.js @@ -1,7 +1,6 @@ /* - * noVNC: HTML5 VNC client - * Copyright (C) 2019 The noVNC Authors - * (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca) + * KasmVNC: HTML5 VNC client + * Copyright (C) 2020 Kasm Technologies * Licensed under MPL 2.0 (see LICENSE.txt) * * See README.md for usage and integration instructions. @@ -15,6 +14,7 @@ export default class UDPDecoder { constructor() { this._filter = null; this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel) + this._directDraw = false; //Draw directly to the canvas without ordering this._zlibs = []; for (let i = 0; i < 4; i++) { @@ -29,20 +29,15 @@ export default class UDPDecoder { let ret; if (ctl === 0x08) { - ret = this._fillRect(x, y, width, height, - data, display, depth); + ret = this._fillRect(x, y, width, height, data, display, depth, frame_id); } else if (ctl === 0x09) { - ret = this._jpegRect(x, y, width, height, - data, display, depth); + ret = this._jpegRect(x, y, width, height, data, display, depth, frame_id); } else if (ctl === 0x0A) { - ret = this._pngRect(x, y, width, height, - data, display, depth); + ret = this._pngRect(x, y, width, height, data, display, depth, frame_id); } else if ((ctl & 0x08) == 0) { - ret = this._basicRect(ctl, x, y, width, height, - data, display, depth); + ret = this._basicRect(ctl, x, y, width, height, data, display, depth, frame_id); } else if (ctl === 0x0B) { - ret = this._webpRect(x, y, width, height, - data, display, depth); + ret = this._webpRect(x, y, width, height, data, display, depth, frame_id); } else { throw new Error("Illegal udp compression received (ctl: " + ctl + ")"); @@ -51,48 +46,48 @@ export default class UDPDecoder { return ret; } - _fillRect(x, y, width, height, data, display, depth) { + _fillRect(x, y, width, height, data, display, depth, frame_id) { display.fillRect(x, y, width, height, - [data[13], data[14], data[15]], false); + [data[13], data[14], data[15]], frame_id, this._directDraw); return true; } - _jpegRect(x, y, width, height, data, display, depth) { + _jpegRect(x, y, width, height, data, display, depth, frame_id) { let img = this._readData(data); if (img === null) { return false; } - display.imageRect(x, y, width, height, "image/jpeg", img); + display.imageRect(x, y, width, height, "image/jpeg", img, frame_id, this._directDraw); return true; } - _webpRect(x, y, width, height, data, display, depth) { + _webpRect(x, y, width, height, data, display, depth, frame_id) { let img = this._readData(data); if (img === null) { return false; } - display.imageRect(x, y, width, height, "image/webp", img); + display.imageRect(x, y, width, height, "image/webp", img, frame_id, this._directDraw); return true; } - _pngRect(x, y, width, height, data, display, depth) { + _pngRect(x, y, width, height, data, display, depth, frame_id) { //throw new Error("PNG received in UDP rect"); Log.Error("PNG received in UDP rect"); } - _basicRect(ctl, x, y, width, height, data, display, depth) { + _basicRect(ctl, x, y, width, height, data, display, depth, frame_id) { let zlibs_flags = data[12]; // Reset streams if the server requests it for (let i = 0; i < 4; i++) { if ((zlibs_flags >> i) & 1) { this._zlibs[i].reset(); - Log.Debug("Reset zlib stream " + i); + //Log.Debug("Reset zlib stream " + i); } } @@ -110,15 +105,15 @@ export default class UDPDecoder { switch (filter) { case 0: // CopyFilter ret = this._copyFilter(streamId, x, y, width, height, - data, display, depth, data_index); + data, display, depth, frame_id, data_index); break; case 1: // PaletteFilter ret = this._paletteFilter(streamId, x, y, width, height, - data, display, depth); + data, display, depth, frame_id); break; case 2: // GradientFilter ret = this._gradientFilter(streamId, x, y, width, height, - data, display, depth); + data, display, depth, frame_id); break; default: throw new Error("Illegal tight filter received (ctl: " + @@ -128,7 +123,7 @@ export default class UDPDecoder { return ret; } - _copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) { + _copyFilter(streamId, x, y, width, height, data, display, depth, frame_id, data_index=14) { const uncompressedSize = width * height * 3; if (uncompressedSize === 0) { @@ -156,12 +151,12 @@ export default class UDPDecoder { rgbx[i + 3] = 255; // Alpha } - display.blitImage(x, y, width, height, rgbx, 0, false); + display.blitImage(x, y, width, height, rgbx, 0, frame_id, this._directDraw); return true; } - _paletteFilter(streamId, x, y, width, height, data, display, depth) { + _paletteFilter(streamId, x, y, width, height, data, display, depth, frame_id) { const numColors = data[14] + 1; const paletteSize = numColors * 3; let palette = data.slice(15, 15 + paletteSize); @@ -190,15 +185,15 @@ export default class UDPDecoder { // Convert indexed (palette based) image data to RGB if (numColors == 2) { - this._monoRect(x, y, width, height, data, palette, display); + this._monoRect(x, y, width, height, data, palette, display, frame_id); } else { - this._paletteRect(x, y, width, height, data, palette, display); + this._paletteRect(x, y, width, height, data, palette, display, frame_id); } return true; } - _monoRect(x, y, width, height, data, palette, display) { + _monoRect(x, y, width, height, data, palette, display, frame_id) { // Convert indexed (palette based) image data to RGB // TODO: reduce number of calculations inside loop const dest = this._getScratchBuffer(width * height * 4); @@ -228,10 +223,10 @@ export default class UDPDecoder { } } - display.blitImage(x, y, width, height, dest, 0, false); + display.blitImage(x, y, width, height, dest, 0, frame_id, this._directDraw); } - _paletteRect(x, y, width, height, data, palette, display) { + _paletteRect(x, y, width, height, data, palette, display, frame_id) { // Convert indexed (palette based) image data to RGB const dest = this._getScratchBuffer(width * height * 4); const total = width * height * 4; @@ -243,10 +238,10 @@ export default class UDPDecoder { dest[i + 3] = 255; } - display.blitImage(x, y, width, height, dest, 0, false); + display.blitImage(x, y, width, height, dest, 0, frame_id, this._directDraw); } - _gradientFilter(streamId, x, y, width, height, data, display, depth) { + _gradientFilter(streamId, x, y, width, height, data, display, depth, frame_id) { throw new Error("Gradient filter not implemented"); } diff --git a/core/display.js b/core/display.js index be4aeaf4..11f6269b 100644 --- a/core/display.js +++ b/core/display.js @@ -1,5 +1,6 @@ /* - * noVNC: HTML5 VNC client + * KasmVNC: HTML5 VNC client + * Copyright (C) 2020 Kasm Technologies * Copyright (C) 2019 The noVNC Authors * Licensed under MPL 2.0 (see LICENSE.txt) * @@ -18,7 +19,7 @@ export default class Display { /* For performance reasons we use a multi dimensional array 1st Dimension of Array Represents Frames, each element is a Frame - 2nd Dimension contains 4 elements + 2nd Dimension is the contents of a frame and meta data, contains 4 elements 0 - int, FrameID 1 - int, Rect Count 2 - Array of Rect objects @@ -26,11 +27,6 @@ export default class Display { 4 - int, index of current rect (post-processing) */ this._asyncFrameQueue = []; - /* - Buffer for incoming frames. The larger the buffer the more time there is to collect, process, and order rects - but the more delay there is. May need to adjust this higher for lower power devices when UDP is complete. - Decoders that use WASM in parallel can also cause out of order rects - */ this._maxAsyncFrameQueue = 3; this._clearAsyncQueue(); @@ -68,13 +64,15 @@ export default class Display { this._lastFlip = Date.now(); this._droppedFrames = 0; this._droppedRects = 0; - this._missingRectCnt = 0; - setInterval(function() { + this._forcedFrameCnt = 0; + this._missingFlipRect = 0; + this._lateFlipRect = 0; + this._frameStatsInterval = setInterval(function() { let delta = Date.now() - this._lastFlip; if (delta > 0) { this._fps = (this._flipCnt / (delta / 1000)).toFixed(2); } - Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Missing Rect Cnt: ' + this._missingRectCnt); + Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect); this._flipCnt = 0; this._lastFlip = Date.now(); }.bind(this), 5000); @@ -91,7 +89,7 @@ export default class Display { this.onflush = () => { }; // A flush request has finished // Use requestAnimationFrame to write to canvas, to match display refresh rate - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); Log.Debug("<< Display.constructor"); } @@ -259,7 +257,11 @@ export default class Display { this.viewportChangePos(0, 0); } - // rendering canvas + /* + * Mark the specified frame with a rect count + * @param {number} frame_id - The frame ID of the target frame + * @param {number} rect_cnt - The number of rects in the target frame + */ flip(frame_id, rect_cnt) { this._asyncRenderQPush({ 'type': 'flip', @@ -268,17 +270,44 @@ export default class Display { }); } + /* + * Is the frame queue full + * @returns {bool} is the queue full + */ pending() { //is the slot in the queue for the newest frame in use return this._asyncFrameQueue[this._maxAsyncFrameQueue - 1][0] > 0; } - flush() { + /* + * Force the oldest frame in the queue to render, whether ready or not. + * @param {bool} onflush_message - The caller wants an onflush event triggered once complete. This is + * useful for TCP, allowing the websocket to block until we are ready to process the next frame. + * UDP cannot block and thus no need to notify the caller when complete. + */ + flush(onflush_message=true) { //force oldest frame to render this._asyncFrameComplete(0, true); - //this in effect blocks more incoming frames until the oldest frame has been rendered to canvas (tcp only) - this._flushing = true; + if (onflush_message) + this._flushing = true; + } + + /* + * Clears the buffer of anything that has not yet been displayed. + * This must be called when switching between transit modes tcp/udp + */ + clear() { + this._clearAsyncQueue(); + } + + /* + * Cleans up resources, should be called on a disconnect + */ + dispose() { + clearInterval(this._frameStatsInterval); + cancelAnimationFrame(this._animationFrameID); + this.clear(); } fillRect(x, y, width, height, color, frame_id, fromQueue) { @@ -432,7 +461,7 @@ export default class Display { let frameIx = -1; let oldestFrameID = Number.MAX_SAFE_INTEGER; let newestFrameID = 0; - for (let i=0; i= 0) { if (rect.type == "flip") { //flip rect contains the rect count for the frame + if (this._asyncFrameQueue[frameIx][1] !== 0) { + Log.Warn("Redundant flip rect, current rect_cnt: " + this._asyncFrameQueue[frameIx][1] + ", new rect_cnt: " + rect.rect_cnt ); + } this._asyncFrameQueue[frameIx][1] = rect.rect_cnt; + if (rect.rect_cnt == 0) { + Log.Warn("Invalid rect count"); + } } if (this._asyncFrameQueue[frameIx][1] == this._asyncFrameQueue[frameIx][2].length) { @@ -464,6 +498,7 @@ export default class Display { if (rect.frame_id < oldestFrameID) { //rect is older than any frame in the queue, drop it this._droppedRects++; + if (rect.type == "flip") { this._lateFlipRect++; } return; } else if (rect.frame_id > newestFrameID) { //frame is newer than any frame in the queue, drop old frames @@ -497,9 +532,12 @@ export default class Display { if (force) { if (this._asyncFrameQueue[frameIx][1] == 0) { - this._missingRectCnt++; + this._missingFlipRect++; //at minimum the flip rect is missing } else if (this._asyncFrameQueue[frameIx][1] !== this._asyncFrameQueue[frameIx][2].length) { this._droppedRects += (this._asyncFrameQueue[frameIx][1] - this._asyncFrameQueue[frameIx][2].length); + if (this._asyncFrameQueue[frameIx][2].length > this._asyncFrameQueue[frameIx][1]) { + Log.Warn("Frame has more rects than the reported rect_cnt."); + } } while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { @@ -525,10 +563,12 @@ export default class Display { /* Push the oldest frame in the buffer to the canvas if it is marked ready */ - _pushAsyncFrame() { + _pushAsyncFrame(force=false) { if (this._asyncFrameQueue[0][3]) { let frame = this._asyncFrameQueue.shift()[2]; - this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]); + if (this._asyncFrameQueue.length < this._maxAsyncFrameQueue) { + this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]); + } //render the selected frame for (let i = 0; i < frame.length; i++) { @@ -560,7 +600,9 @@ export default class Display { } } - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + if (!force) { + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + } } _rescale(factor) { diff --git a/core/rfb.js b/core/rfb.js index 063603a7..df62822a 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1,5 +1,6 @@ /* - * noVNC: HTML5 VNC client + * KasmVNC: HTML5 VNC client + * Copyright (C) 2020 Kasm Technologies * Copyright (C) 2020 The noVNC Authors * Licensed under MPL 2.0 (see LICENSE.txt) * @@ -1107,20 +1108,25 @@ export default class RFB extends EventTargetMixin { (u8[14] << 16) + (u8[15] << 24), 10); // TODO: check the hash. It's the low 32 bits of XXH64, seed 0 + const frame_id = parseInt(u8[16] + + (u8[17] << 8) + + (u8[18] << 16) + + (u8[19] << 24), 10); if (me._transitConnectionState !== me.TransitConnectionStates.Udp) { + me._display.clear(); me._changeTransitConnectionState(me.TransitConnectionStates.Udp); } if (pieces == 1) { // Handle it immediately - me._handleUdpRect(u8.slice(16)); - } else { // Insert into wait array + me._handleUdpRect(u8.slice(20), frame_id); + } else { // Use buffer const now = Date.now(); if (udpBuffer.has(id)) { let item = udpBuffer.get(id); item.recieved_pieces += 1; - item.data[i] = u8.slice(16); + item.data[i] = u8.slice(20); item.total_bytes += item.data[i].length; if (item.total_pieces == item.recieved_pieces) { @@ -1132,7 +1138,7 @@ export default class RFB extends EventTargetMixin { z += item.data[x].length; } udpBuffer.delete(id); - me._handleUdpRect(finaldata); + me._handleUdpRect(finaldata, frame_id); } } else { let item = { @@ -1142,7 +1148,7 @@ export default class RFB extends EventTargetMixin { total_bytes: 0, // total size of all data pieces combined data: new Array(pieces) } - item.data[i] = u8.slice(16); + item.data[i] = u8.slice(20); item.total_bytes = item.data[i].length; udpBuffer.set(id, item); } @@ -1192,6 +1198,7 @@ export default class RFB extends EventTargetMixin { throw e; } } + this._display.dispose(); clearTimeout(this._resizeTimeout); clearTimeout(this._mouseMoveTimer); Log.Debug("<< RFB.disconnect"); @@ -3054,7 +3061,7 @@ export default class RFB extends EventTargetMixin { } } - _handleUdpRect(data) { + _handleUdpRect(data, frame_id) { let frame = { x: (data[0] << 8) + data[1], y: (data[2] << 8) + data[3], @@ -3066,10 +3073,9 @@ export default class RFB extends EventTargetMixin { switch (frame.encoding) { case encodings.pseudoEncodingLastRect: - if (document.visibilityState !== "hidden") { - this._display.flip(false); //TODO: UDP is now broken, flip needs rect count and frame number - this._udpBuffer.clear(); - } + this._display.flip(frame_id, frame.x + 1); //Last Rect message, first 16 bytes contain rect count + if (this._display.pending()) + this._display.flush(false); break; case encodings.encodingTight: let decoder = this._decoders[encodings.encodingUDP]; @@ -3077,7 +3083,7 @@ export default class RFB extends EventTargetMixin { decoder.decodeRect(frame.x, frame.y, frame.width, frame.height, data, this._display, - this._fbDepth); + this._fbDepth, frame_id); } catch (err) { this._fail("Error decoding rect: " + err); return false; @@ -3203,7 +3209,7 @@ export default class RFB extends EventTargetMixin { this._FBU.encoding = null; } - if (this._FBU.rect_total > 0) { + if (this._FBU.rect_total > 1) { this._display.flip(this._FBU.frame_id, this._FBU.rect_total); } @@ -3528,6 +3534,7 @@ export default class RFB extends EventTargetMixin { this._udpTransitFailures++; } this._changeTransitConnectionState(this.TransitConnectionStates.Tcp); + this._display.clear(); if (this._useUdp) { if (this._udpConnectFailures < 3 && this._udpTransitFailures < 3) { setTimeout(function() { @@ -3539,6 +3546,7 @@ export default class RFB extends EventTargetMixin { } } } else if (this._transitConnectionState == this.TransitConnectionStates.Downgrading) { + this._display.clear(); this._changeTransitConnectionState(this.TransitConnectionStates.Tcp); } return decoder.decodeRect(this._FBU.x, this._FBU.y,