Feature/kasm 3380 udp frame (#46)

* Handle the frame id in udp packets

* refactor udp message buffer, add frame_id and rect cnt

* refactor udp to work with new display.js

* additional debug metrics, clear frame buffer on transitions tcp/udp

* fix udp with display refactor, KASM-3541 cancelAnimationFrame

Co-authored-by: Lauri Kasanen <cand@gmx.com>
Co-authored-by: mattmcclaskey <matt@kasmweb.com>
This commit is contained in:
Matt McClaskey 2022-11-11 09:31:16 -05:00 committed by GitHub
parent 3a6a63cde3
commit 437830d497
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 118 additions and 71 deletions

View File

@ -1,5 +1,6 @@
/*
* noVNC: HTML5 VNC client
* KasmVNC: HTML5 VNC client
* Copyright (C) 2020 Kasm Technologies
* Copyright (C) 2019 The noVNC Authors
* Licensed under MPL 2.0 (see LICENSE.txt)
*

View File

@ -1,5 +1,6 @@
/*
* noVNC: HTML5 VNC client
* KasmVNC: HTML5 VNC client
* Copyright (C) 2020 Kasm Technologies
* Copyright (C) 2019 The noVNC Authors
* (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca)
* Licensed under MPL 2.0 (see LICENSE.txt)

View File

@ -1,7 +1,6 @@
/*
* noVNC: HTML5 VNC client
* Copyright (C) 2019 The noVNC Authors
* (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca)
* KasmVNC: HTML5 VNC client
* Copyright (C) 2020 Kasm Technologies
* Licensed under MPL 2.0 (see LICENSE.txt)
*
* See README.md for usage and integration instructions.
@ -15,6 +14,7 @@ export default class UDPDecoder {
constructor() {
this._filter = null;
this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel)
this._directDraw = false; //Draw directly to the canvas without ordering
this._zlibs = [];
for (let i = 0; i < 4; i++) {
@ -29,20 +29,15 @@ export default class UDPDecoder {
let ret;
if (ctl === 0x08) {
ret = this._fillRect(x, y, width, height,
data, display, depth);
ret = this._fillRect(x, y, width, height, data, display, depth, frame_id);
} else if (ctl === 0x09) {
ret = this._jpegRect(x, y, width, height,
data, display, depth);
ret = this._jpegRect(x, y, width, height, data, display, depth, frame_id);
} else if (ctl === 0x0A) {
ret = this._pngRect(x, y, width, height,
data, display, depth);
ret = this._pngRect(x, y, width, height, data, display, depth, frame_id);
} else if ((ctl & 0x08) == 0) {
ret = this._basicRect(ctl, x, y, width, height,
data, display, depth);
ret = this._basicRect(ctl, x, y, width, height, data, display, depth, frame_id);
} else if (ctl === 0x0B) {
ret = this._webpRect(x, y, width, height,
data, display, depth);
ret = this._webpRect(x, y, width, height, data, display, depth, frame_id);
} else {
throw new Error("Illegal udp compression received (ctl: " +
ctl + ")");
@ -51,48 +46,48 @@ export default class UDPDecoder {
return ret;
}
_fillRect(x, y, width, height, data, display, depth) {
_fillRect(x, y, width, height, data, display, depth, frame_id) {
display.fillRect(x, y, width, height,
[data[13], data[14], data[15]], false);
[data[13], data[14], data[15]], frame_id, this._directDraw);
return true;
}
_jpegRect(x, y, width, height, data, display, depth) {
_jpegRect(x, y, width, height, data, display, depth, frame_id) {
let img = this._readData(data);
if (img === null) {
return false;
}
display.imageRect(x, y, width, height, "image/jpeg", img);
display.imageRect(x, y, width, height, "image/jpeg", img, frame_id, this._directDraw);
return true;
}
_webpRect(x, y, width, height, data, display, depth) {
_webpRect(x, y, width, height, data, display, depth, frame_id) {
let img = this._readData(data);
if (img === null) {
return false;
}
display.imageRect(x, y, width, height, "image/webp", img);
display.imageRect(x, y, width, height, "image/webp", img, frame_id, this._directDraw);
return true;
}
_pngRect(x, y, width, height, data, display, depth) {
_pngRect(x, y, width, height, data, display, depth, frame_id) {
//throw new Error("PNG received in UDP rect");
Log.Error("PNG received in UDP rect");
}
_basicRect(ctl, x, y, width, height, data, display, depth) {
_basicRect(ctl, x, y, width, height, data, display, depth, frame_id) {
let zlibs_flags = data[12];
// Reset streams if the server requests it
for (let i = 0; i < 4; i++) {
if ((zlibs_flags >> i) & 1) {
this._zlibs[i].reset();
Log.Debug("Reset zlib stream " + i);
//Log.Debug("Reset zlib stream " + i);
}
}
@ -110,15 +105,15 @@ export default class UDPDecoder {
switch (filter) {
case 0: // CopyFilter
ret = this._copyFilter(streamId, x, y, width, height,
data, display, depth, data_index);
data, display, depth, frame_id, data_index);
break;
case 1: // PaletteFilter
ret = this._paletteFilter(streamId, x, y, width, height,
data, display, depth);
data, display, depth, frame_id);
break;
case 2: // GradientFilter
ret = this._gradientFilter(streamId, x, y, width, height,
data, display, depth);
data, display, depth, frame_id);
break;
default:
throw new Error("Illegal tight filter received (ctl: " +
@ -128,7 +123,7 @@ export default class UDPDecoder {
return ret;
}
_copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) {
_copyFilter(streamId, x, y, width, height, data, display, depth, frame_id, data_index=14) {
const uncompressedSize = width * height * 3;
if (uncompressedSize === 0) {
@ -156,12 +151,12 @@ export default class UDPDecoder {
rgbx[i + 3] = 255; // Alpha
}
display.blitImage(x, y, width, height, rgbx, 0, false);
display.blitImage(x, y, width, height, rgbx, 0, frame_id, this._directDraw);
return true;
}
_paletteFilter(streamId, x, y, width, height, data, display, depth) {
_paletteFilter(streamId, x, y, width, height, data, display, depth, frame_id) {
const numColors = data[14] + 1;
const paletteSize = numColors * 3;
let palette = data.slice(15, 15 + paletteSize);
@ -190,15 +185,15 @@ export default class UDPDecoder {
// Convert indexed (palette based) image data to RGB
if (numColors == 2) {
this._monoRect(x, y, width, height, data, palette, display);
this._monoRect(x, y, width, height, data, palette, display, frame_id);
} else {
this._paletteRect(x, y, width, height, data, palette, display);
this._paletteRect(x, y, width, height, data, palette, display, frame_id);
}
return true;
}
_monoRect(x, y, width, height, data, palette, display) {
_monoRect(x, y, width, height, data, palette, display, frame_id) {
// Convert indexed (palette based) image data to RGB
// TODO: reduce number of calculations inside loop
const dest = this._getScratchBuffer(width * height * 4);
@ -228,10 +223,10 @@ export default class UDPDecoder {
}
}
display.blitImage(x, y, width, height, dest, 0, false);
display.blitImage(x, y, width, height, dest, 0, frame_id, this._directDraw);
}
_paletteRect(x, y, width, height, data, palette, display) {
_paletteRect(x, y, width, height, data, palette, display, frame_id) {
// Convert indexed (palette based) image data to RGB
const dest = this._getScratchBuffer(width * height * 4);
const total = width * height * 4;
@ -243,10 +238,10 @@ export default class UDPDecoder {
dest[i + 3] = 255;
}
display.blitImage(x, y, width, height, dest, 0, false);
display.blitImage(x, y, width, height, dest, 0, frame_id, this._directDraw);
}
_gradientFilter(streamId, x, y, width, height, data, display, depth) {
_gradientFilter(streamId, x, y, width, height, data, display, depth, frame_id) {
throw new Error("Gradient filter not implemented");
}

View File

@ -1,5 +1,6 @@
/*
* noVNC: HTML5 VNC client
* KasmVNC: HTML5 VNC client
* Copyright (C) 2020 Kasm Technologies
* Copyright (C) 2019 The noVNC Authors
* Licensed under MPL 2.0 (see LICENSE.txt)
*
@ -18,7 +19,7 @@ export default class Display {
/*
For performance reasons we use a multi dimensional array
1st Dimension of Array Represents Frames, each element is a Frame
2nd Dimension contains 4 elements
2nd Dimension is the contents of a frame and meta data, contains 4 elements
0 - int, FrameID
1 - int, Rect Count
2 - Array of Rect objects
@ -26,11 +27,6 @@ export default class Display {
4 - int, index of current rect (post-processing)
*/
this._asyncFrameQueue = [];
/*
Buffer for incoming frames. The larger the buffer the more time there is to collect, process, and order rects
but the more delay there is. May need to adjust this higher for lower power devices when UDP is complete.
Decoders that use WASM in parallel can also cause out of order rects
*/
this._maxAsyncFrameQueue = 3;
this._clearAsyncQueue();
@ -68,13 +64,15 @@ export default class Display {
this._lastFlip = Date.now();
this._droppedFrames = 0;
this._droppedRects = 0;
this._missingRectCnt = 0;
setInterval(function() {
this._forcedFrameCnt = 0;
this._missingFlipRect = 0;
this._lateFlipRect = 0;
this._frameStatsInterval = setInterval(function() {
let delta = Date.now() - this._lastFlip;
if (delta > 0) {
this._fps = (this._flipCnt / (delta / 1000)).toFixed(2);
}
Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Missing Rect Cnt: ' + this._missingRectCnt);
Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect);
this._flipCnt = 0;
this._lastFlip = Date.now();
}.bind(this), 5000);
@ -91,7 +89,7 @@ export default class Display {
this.onflush = () => { }; // A flush request has finished
// Use requestAnimationFrame to write to canvas, to match display refresh rate
window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
Log.Debug("<< Display.constructor");
}
@ -259,7 +257,11 @@ export default class Display {
this.viewportChangePos(0, 0);
}
// rendering canvas
/*
* Mark the specified frame with a rect count
* @param {number} frame_id - The frame ID of the target frame
* @param {number} rect_cnt - The number of rects in the target frame
*/
flip(frame_id, rect_cnt) {
this._asyncRenderQPush({
'type': 'flip',
@ -268,17 +270,44 @@ export default class Display {
});
}
/*
* Is the frame queue full
* @returns {bool} is the queue full
*/
pending() {
//is the slot in the queue for the newest frame in use
return this._asyncFrameQueue[this._maxAsyncFrameQueue - 1][0] > 0;
}
flush() {
/*
* Force the oldest frame in the queue to render, whether ready or not.
* @param {bool} onflush_message - The caller wants an onflush event triggered once complete. This is
* useful for TCP, allowing the websocket to block until we are ready to process the next frame.
* UDP cannot block and thus no need to notify the caller when complete.
*/
flush(onflush_message=true) {
//force oldest frame to render
this._asyncFrameComplete(0, true);
//this in effect blocks more incoming frames until the oldest frame has been rendered to canvas (tcp only)
this._flushing = true;
if (onflush_message)
this._flushing = true;
}
/*
* Clears the buffer of anything that has not yet been displayed.
* This must be called when switching between transit modes tcp/udp
*/
clear() {
this._clearAsyncQueue();
}
/*
* Cleans up resources, should be called on a disconnect
*/
dispose() {
clearInterval(this._frameStatsInterval);
cancelAnimationFrame(this._animationFrameID);
this.clear();
}
fillRect(x, y, width, height, color, frame_id, fromQueue) {
@ -432,7 +461,7 @@ export default class Display {
let frameIx = -1;
let oldestFrameID = Number.MAX_SAFE_INTEGER;
let newestFrameID = 0;
for (let i=0; i<this._maxAsyncFrameQueue; i++) {
for (let i=0; i<this._asyncFrameQueue.length; i++) {
if (rect.frame_id == this._asyncFrameQueue[i][0]) {
this._asyncFrameQueue[i][2].push(rect);
frameIx = i;
@ -440,7 +469,6 @@ export default class Display {
} else if (this._asyncFrameQueue[i][0] == 0) {
let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0);
this._asyncFrameQueue[i][0] = rect.frame_id;
this._asyncFrameQueue[i][1] = rect_cnt;
this._asyncFrameQueue[i][2].push(rect);
this._asyncFrameQueue[i][3] = (rect_cnt == 1);
frameIx = i;
@ -453,7 +481,13 @@ export default class Display {
if (frameIx >= 0) {
if (rect.type == "flip") {
//flip rect contains the rect count for the frame
if (this._asyncFrameQueue[frameIx][1] !== 0) {
Log.Warn("Redundant flip rect, current rect_cnt: " + this._asyncFrameQueue[frameIx][1] + ", new rect_cnt: " + rect.rect_cnt );
}
this._asyncFrameQueue[frameIx][1] = rect.rect_cnt;
if (rect.rect_cnt == 0) {
Log.Warn("Invalid rect count");
}
}
if (this._asyncFrameQueue[frameIx][1] == this._asyncFrameQueue[frameIx][2].length) {
@ -464,6 +498,7 @@ export default class Display {
if (rect.frame_id < oldestFrameID) {
//rect is older than any frame in the queue, drop it
this._droppedRects++;
if (rect.type == "flip") { this._lateFlipRect++; }
return;
} else if (rect.frame_id > newestFrameID) {
//frame is newer than any frame in the queue, drop old frames
@ -497,9 +532,12 @@ export default class Display {
if (force) {
if (this._asyncFrameQueue[frameIx][1] == 0) {
this._missingRectCnt++;
this._missingFlipRect++; //at minimum the flip rect is missing
} else if (this._asyncFrameQueue[frameIx][1] !== this._asyncFrameQueue[frameIx][2].length) {
this._droppedRects += (this._asyncFrameQueue[frameIx][1] - this._asyncFrameQueue[frameIx][2].length);
if (this._asyncFrameQueue[frameIx][2].length > this._asyncFrameQueue[frameIx][1]) {
Log.Warn("Frame has more rects than the reported rect_cnt.");
}
}
while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) {
if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
@ -525,10 +563,12 @@ export default class Display {
/*
Push the oldest frame in the buffer to the canvas if it is marked ready
*/
_pushAsyncFrame() {
_pushAsyncFrame(force=false) {
if (this._asyncFrameQueue[0][3]) {
let frame = this._asyncFrameQueue.shift()[2];
this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]);
if (this._asyncFrameQueue.length < this._maxAsyncFrameQueue) {
this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]);
}
//render the selected frame
for (let i = 0; i < frame.length; i++) {
@ -560,7 +600,9 @@ export default class Display {
}
}
window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
if (!force) {
window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
}
}
_rescale(factor) {

View File

@ -1,5 +1,6 @@
/*
* noVNC: HTML5 VNC client
* KasmVNC: HTML5 VNC client
* Copyright (C) 2020 Kasm Technologies
* Copyright (C) 2020 The noVNC Authors
* Licensed under MPL 2.0 (see LICENSE.txt)
*
@ -1107,20 +1108,25 @@ export default class RFB extends EventTargetMixin {
(u8[14] << 16) +
(u8[15] << 24), 10);
// TODO: check the hash. It's the low 32 bits of XXH64, seed 0
const frame_id = parseInt(u8[16] +
(u8[17] << 8) +
(u8[18] << 16) +
(u8[19] << 24), 10);
if (me._transitConnectionState !== me.TransitConnectionStates.Udp) {
me._display.clear();
me._changeTransitConnectionState(me.TransitConnectionStates.Udp);
}
if (pieces == 1) { // Handle it immediately
me._handleUdpRect(u8.slice(16));
} else { // Insert into wait array
me._handleUdpRect(u8.slice(20), frame_id);
} else { // Use buffer
const now = Date.now();
if (udpBuffer.has(id)) {
let item = udpBuffer.get(id);
item.recieved_pieces += 1;
item.data[i] = u8.slice(16);
item.data[i] = u8.slice(20);
item.total_bytes += item.data[i].length;
if (item.total_pieces == item.recieved_pieces) {
@ -1132,7 +1138,7 @@ export default class RFB extends EventTargetMixin {
z += item.data[x].length;
}
udpBuffer.delete(id);
me._handleUdpRect(finaldata);
me._handleUdpRect(finaldata, frame_id);
}
} else {
let item = {
@ -1142,7 +1148,7 @@ export default class RFB extends EventTargetMixin {
total_bytes: 0, // total size of all data pieces combined
data: new Array(pieces)
}
item.data[i] = u8.slice(16);
item.data[i] = u8.slice(20);
item.total_bytes = item.data[i].length;
udpBuffer.set(id, item);
}
@ -1192,6 +1198,7 @@ export default class RFB extends EventTargetMixin {
throw e;
}
}
this._display.dispose();
clearTimeout(this._resizeTimeout);
clearTimeout(this._mouseMoveTimer);
Log.Debug("<< RFB.disconnect");
@ -3054,7 +3061,7 @@ export default class RFB extends EventTargetMixin {
}
}
_handleUdpRect(data) {
_handleUdpRect(data, frame_id) {
let frame = {
x: (data[0] << 8) + data[1],
y: (data[2] << 8) + data[3],
@ -3066,10 +3073,9 @@ export default class RFB extends EventTargetMixin {
switch (frame.encoding) {
case encodings.pseudoEncodingLastRect:
if (document.visibilityState !== "hidden") {
this._display.flip(false); //TODO: UDP is now broken, flip needs rect count and frame number
this._udpBuffer.clear();
}
this._display.flip(frame_id, frame.x + 1); //Last Rect message, first 16 bytes contain rect count
if (this._display.pending())
this._display.flush(false);
break;
case encodings.encodingTight:
let decoder = this._decoders[encodings.encodingUDP];
@ -3077,7 +3083,7 @@ export default class RFB extends EventTargetMixin {
decoder.decodeRect(frame.x, frame.y,
frame.width, frame.height,
data, this._display,
this._fbDepth);
this._fbDepth, frame_id);
} catch (err) {
this._fail("Error decoding rect: " + err);
return false;
@ -3203,7 +3209,7 @@ export default class RFB extends EventTargetMixin {
this._FBU.encoding = null;
}
if (this._FBU.rect_total > 0) {
if (this._FBU.rect_total > 1) {
this._display.flip(this._FBU.frame_id, this._FBU.rect_total);
}
@ -3528,6 +3534,7 @@ export default class RFB extends EventTargetMixin {
this._udpTransitFailures++;
}
this._changeTransitConnectionState(this.TransitConnectionStates.Tcp);
this._display.clear();
if (this._useUdp) {
if (this._udpConnectFailures < 3 && this._udpTransitFailures < 3) {
setTimeout(function() {
@ -3539,6 +3546,7 @@ export default class RFB extends EventTargetMixin {
}
}
} else if (this._transitConnectionState == this.TransitConnectionStates.Downgrading) {
this._display.clear();
this._changeTransitConnectionState(this.TransitConnectionStates.Tcp);
}
return decoder.decodeRect(this._FBU.x, this._FBU.y,