jcarr
/
noVNC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

KASM-3367 Refactor display.js, direct to visible canvas (#40) 

* refactor display.js, remove non-visible canvas and concept of damage

* tweaks to UDP transitions

* Update testing readme

Co-authored-by: matt <matt@kasmweb.com>
Co-authored-by: matt mcclaskey <matt@kamsweb.com>
This commit is contained in:
Matt McClaskey 2022-10-14 15:07:57 -04:00 committed by GitHub
parent 26c97c1dd3
commit 9a95983382
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
core/display.js
View File

@ -12,9 +12,9 @@ import { toSigned32bit } from './util/int.js';
export default class Display { export default class Display {
constructor(target) { constructor(target) {
this._drawCtx = null;
this._renderQ = []; // queue drawing actions for in-oder rendering this._renderQ = []; // queue drawing actions for in-oder rendering
this._currentFrame = [];
this._nextFrame = [];
this._flushing = false; this._flushing = false;
// the full frame buffer (logical canvas) size // the full frame buffer (logical canvas) size
@ -47,27 +47,18 @@ export default class Display {
// the visible canvas viewport (i.e. what actually gets seen) // the visible canvas viewport (i.e. what actually gets seen)
this._viewportLoc = { 'x': 0, 'y': 0, 'w': this._target.width, 'h': this._target.height }; this._viewportLoc = { 'x': 0, 'y': 0, 'w': this._target.width, 'h': this._target.height };
// The hidden canvas, where we do the actual rendering
this._backbuffer = document.createElement('canvas');
this._drawCtx = this._backbuffer.getContext('2d');
this._damageBounds = { left: 0, top: 0,
right: this._backbuffer.width,
bottom: this._backbuffer.height };
Log.Debug("User Agent: " + navigator.userAgent); Log.Debug("User Agent: " + navigator.userAgent);
// performance metrics, try to calc a fps equivelant // performance metrics, try to calc a fps equivelant
this._flipCnt = 0; this._flipCnt = 0;
this._currentFrameDamages = [];
this._lastFlip = Date.now(); this._lastFlip = Date.now();
setInterval(function() { setInterval(function() {
let delta = Date.now() - this._lastFlip; let delta = Date.now() - this._lastFlip;
if (delta > 0) { if (delta > 0) {
this._fps = (this._flipCnt / (delta / 1000)).toFixed(2); this._fps = (this._flipCnt / (delta / 1000)).toFixed(2);
} }
this._lastFlip = Date.now();
this._flipCnt = 0; this._flipCnt = 0;
this._lastFlip = Date.now();
}.bind(this), 5000); }.bind(this), 5000);
Log.Debug("<< Display.constructor"); Log.Debug("<< Display.constructor");
@ -159,11 +150,6 @@ export default class Display {
} }
Log.Debug("viewportChange deltaX: " + deltaX + ", deltaY: " + deltaY); Log.Debug("viewportChange deltaX: " + deltaX + ", deltaY: " + deltaY);
vp.x += deltaX;
vp.y += deltaY;
this._damage(vp.x, vp.y, vp.w, vp.h);
this.flip(); this.flip();
} }
@ -200,7 +186,6 @@ export default class Display {
// The position might need to be updated if we've grown // The position might need to be updated if we've grown
this.viewportChangePos(0, 0); this.viewportChangePos(0, 0);
this._damage(vp.x, vp.y, vp.w, vp.h);
this.flip(); this.flip();
// Update the visible size of the target canvas // Update the visible size of the target canvas
@ -228,13 +213,14 @@ export default class Display {
this._fbWidth = width; this._fbWidth = width;
this._fbHeight = height; this._fbHeight = height;
const canvas = this._backbuffer; const canvas = this._target;
if (canvas == undefined) { return; }
if (canvas.width !== width || canvas.height !== height) { if (canvas.width !== width || canvas.height !== height) {
// We have to save the canvas data since changing the size will clear it // We have to save the canvas data since changing the size will clear it
let saveImg = null; let saveImg = null;
if (canvas.width > 0 && canvas.height > 0) { if (canvas.width > 0 && canvas.height > 0) {
saveImg = this._drawCtx.getImageData(0, 0, canvas.width, canvas.height); saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height);
} }
if (canvas.width !== width) { if (canvas.width !== width) {
@ -245,7 +231,7 @@ export default class Display {
} }
if (saveImg) { if (saveImg) {
this._drawCtx.putImageData(saveImg, 0, 0); this._targetCtx.putImageData(saveImg, 0, 0);
} }
} }
@ -256,85 +242,32 @@ export default class Display {
this.viewportChangePos(0, 0); this.viewportChangePos(0, 0);
} }
// Track what parts of the visible canvas that need updating
_damage(x, y, w, h) {
if (x < this._damageBounds.left) {
this._damageBounds.left = x;
}
if (y < this._damageBounds.top) {
this._damageBounds.top = y;
}
if ((x + w) > this._damageBounds.right) {
this._damageBounds.right = x + w;
}
if ((y + h) > this._damageBounds.bottom) {
this._damageBounds.bottom = y + h;
}
}
// Attempt to determine when updates overlap an area and thus indicate a new frame
isNewFrame(x, y, w, h) {
for (var i = 0; i < this._currentFrameDamages.length; i++) {
let area = this._currentFrameDamages[i];
if (x >= area.x && x <= (area.x + area.w) && y >= area.y && y <= (area.y + area.h)) {
this._currentFrameDamages = [];
return true;
}
}
var new_area = { x: x, y: y, w: w, h: h }
this._currentFrameDamages.push(new_area);
return false;
}
// Update the visible canvas with the contents of the
// rendering canvas // rendering canvas
flip(fromQueue) { flip(fromQueue) {
if (this._renderQ.length !== 0 && !fromQueue) { if (!fromQueue) {
this._renderQPush({ this._renderQPush({
'type': 'flip' 'type': 'flip'
}); });
} else { } else {
let x = this._damageBounds.left; for (let i = 0; i < this._currentFrame.length; i++) {
let y = this._damageBounds.top; const a = this._currentFrame[i];
let w = this._damageBounds.right - x; switch (a.type) {
let h = this._damageBounds.bottom - y; case 'copy':
this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, true);
let vx = x - this._viewportLoc.x; break;
let vy = y - this._viewportLoc.y; case 'fill':
this.fillRect(a.x, a.y, a.width, a.height, a.color, true);
if (vx < 0) { break;
w += vx; case 'blit':
x -= vx; this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, true);
vx = 0; break;
case 'img':
this.drawImage(a.img, a.x, a.y, a.width, a.height);
break;
} }
if (vy < 0) {
h += vy;
y -= vy;
vy = 0;
} }
if ((vx + w) > this._viewportLoc.w) {
w = this._viewportLoc.w - vx;
}
if ((vy + h) > this._viewportLoc.h) {
h = this._viewportLoc.h - vy;
}
if ((w > 0) && (h > 0)) {
// FIXME: We may need to disable image smoothing here
// as well (see copyImage()), but we haven't
// noticed any problem yet.
this._targetCtx.drawImage(this._backbuffer,
x, y, w, h,
vx, vy, w, h);
this._flipCnt += 1; this._flipCnt += 1;
} }
this._damageBounds.left = this._damageBounds.top = 65535;
this._damageBounds.right = this._damageBounds.bottom = 0;
}
} }
pending() { pending() {
@ -350,7 +283,7 @@ export default class Display {
} }
fillRect(x, y, width, height, color, fromQueue) { fillRect(x, y, width, height, color, fromQueue) {
if (this._renderQ.length !== 0 && !fromQueue) { if (!fromQueue) {
this._renderQPush({ this._renderQPush({
'type': 'fill', 'type': 'fill',
'x': x, 'x': x,
@ -361,13 +294,12 @@ export default class Display {
}); });
} else { } else {
this._setFillColor(color); this._setFillColor(color);
this._drawCtx.fillRect(x, y, width, height); this._targetCtx.fillRect(x, y, width, height);
this._damage(x, y, width, height);
} }
} }
copyImage(oldX, oldY, newX, newY, w, h, fromQueue) { copyImage(oldX, oldY, newX, newY, w, h, fromQueue) {
if (this._renderQ.length !== 0 && !fromQueue) { if (!fromQueue) {
this._renderQPush({ this._renderQPush({
'type': 'copy', 'type': 'copy',
'oldX': oldX, 'oldX': oldX,
@ -385,15 +317,14 @@ export default class Display {
// //
// We need to set these every time since all properties are reset // We need to set these every time since all properties are reset
// when the the size is changed // when the the size is changed
this._drawCtx.mozImageSmoothingEnabled = false; this._targetCtx.mozImageSmoothingEnabled = false;
this._drawCtx.webkitImageSmoothingEnabled = false; this._targetCtx.webkitImageSmoothingEnabled = false;
this._drawCtx.msImageSmoothingEnabled = false; this._targetCtx.msImageSmoothingEnabled = false;
this._drawCtx.imageSmoothingEnabled = false; this._targetCtx.imageSmoothingEnabled = false;
this._drawCtx.drawImage(this._backbuffer, this._targetCtx.drawImage(this._target,
oldX, oldY, w, h, oldX, oldY, w, h,
newX, newY, w, h); newX, newY, w, h);
this._damage(newX, newY, w, h);
} }
} }
@ -417,7 +348,7 @@ export default class Display {
} }
blitImage(x, y, width, height, arr, offset, fromQueue) { blitImage(x, y, width, height, arr, offset, fromQueue) {
if (this._renderQ.length !== 0 && !fromQueue) { if (!fromQueue) {
// NB(directxman12): it's technically more performant here to use preallocated arrays, // NB(directxman12): it's technically more performant here to use preallocated arrays,
// but it's a lot of extra work for not a lot of payoff -- if we're using the render queue, // but it's a lot of extra work for not a lot of payoff -- if we're using the render queue,
// this probably isn't getting called *nearly* as much // this probably isn't getting called *nearly* as much
@ -437,22 +368,20 @@ export default class Display {
arr.byteOffset + offset, arr.byteOffset + offset,
width * height * 4); width * height * 4);
let img = new ImageData(data, width, height); let img = new ImageData(data, width, height);
this._drawCtx.putImageData(img, x, y); this._targetCtx.putImageData(img, x, y);
this._damage(x, y, width, height);
} }
} }
drawImage(img, x, y, w, h) { drawImage(img, x, y, w, h) {
try { try {
if (img.width != w || img.height != h) { if (img.width != w || img.height != h) {
this._drawCtx.drawImage(img, x, y, w, h); this._targetCtx.drawImage(img, x, y, w, h);
} else { } else {
this._drawCtx.drawImage(img, x, y); this._targetCtx.drawImage(img, x, y);
} }
} catch (error) { } catch (error) {
Log.Error('Invalid image recieved.'); //KASM-2090 Log.Error('Invalid image recieved.'); //KASM-2090
} }
this._damage(x, y, w, h);
} }
autoscale(containerWidth, containerHeight, scaleRatio=0) { autoscale(containerWidth, containerHeight, scaleRatio=0) {
@ -512,7 +441,7 @@ export default class Display {
_setFillColor(color) { _setFillColor(color) {
const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')'; const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')';
if (newStyle !== this._prevDrawStyle) { if (newStyle !== this._prevDrawStyle) {
this._drawCtx.fillStyle = newStyle; this._targetCtx.fillStyle = newStyle;
this._prevDrawStyle = newStyle; this._prevDrawStyle = newStyle;
} }
} }
@ -520,15 +449,11 @@ export default class Display {
_renderQPush(action) { _renderQPush(action) {
this._renderQ.push(action); this._renderQ.push(action);
if (this._renderQ.length === 1) { if (this._renderQ.length === 1) {
// If this can be rendered immediately it will be, otherwise
// the scanner will wait for the relevant event
this._scanRenderQ(); this._scanRenderQ();
} }
} }
_resumeRenderQ() { _resumeRenderQ() {
// "this" is the object that is ready, not the
// display object
this.removeEventListener('load', this._noVNCDisplay._resumeRenderQ); this.removeEventListener('load', this._noVNCDisplay._resumeRenderQ);
this._noVNCDisplay._scanRenderQ(); this._noVNCDisplay._scanRenderQ();
} }
@ -540,34 +465,23 @@ export default class Display {
const a = this._renderQ[0]; const a = this._renderQ[0];
switch (a.type) { switch (a.type) {
case 'flip': case 'flip':
this._currentFrame = this._nextFrame;
this._nextFrame = [];
this.flip(true); this.flip(true);
break; break;
case 'copy':
this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, true);
break;
case 'fill':
this.fillRect(a.x, a.y, a.width, a.height, a.color, true);
break;
case 'blit':
this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, true);
break;
case 'img': case 'img':
if (a.img.complete) { if (a.img.complete) {
/* if (a.img.width !== a.width || a.img.height !== a.height) { this._nextFrame.push(a);
Log.Error("Decoded image has incorrect dimensions. Got " +
a.img.width + "x" + a.img.height + ". Expected " +
a.width + "x" + a.height + ".");
return;
}*/
this.drawImage(a.img, a.x, a.y, a.width, a.height);
} else { } else {
a.img._noVNCDisplay = this; a.img._noVNCDisplay = this;
a.img.addEventListener('load', this._resumeRenderQ); a.img.addEventListener('load', this._resumeRenderQ);
// We need to wait for this image to 'load' // We need to wait for this image to 'load'
// to keep things in-order
ready = false; ready = false;
} }
break; break;
default:
this._nextFrame.push(a);
break;
} }
if (ready) { if (ready) {

17
core/rfb.js
View File

@ -147,6 +147,7 @@ export default class RFB extends EventTargetMixin {
Failure: Symbol("failure") Failure: Symbol("failure")
} }
this._transitConnectionState = this.TransitConnectionStates.Tcp; this._transitConnectionState = this.TransitConnectionStates.Tcp;
this._lastTransition = null;
this._udpConnectFailures = 0; //Failures in upgrading connection to udp this._udpConnectFailures = 0; //Failures in upgrading connection to udp
this._udpTransitFailures = 0; //Failures in transit after successful upgrade this._udpTransitFailures = 0; //Failures in transit after successful upgrade
@ -230,6 +231,7 @@ export default class RFB extends EventTargetMixin {
this._canvas.width = 0; this._canvas.width = 0;
this._canvas.height = 0; this._canvas.height = 0;
this._canvas.tabIndex = -1; this._canvas.tabIndex = -1;
this._canvas.overflow = 'hidden';
this._screen.appendChild(this._canvas); this._screen.appendChild(this._canvas);
// Cursor // Cursor
@ -714,11 +716,11 @@ export default class RFB extends EventTargetMixin {
set enableWebRTC(value) { set enableWebRTC(value) {
this._useUdp = value; this._useUdp = value;
if (!value) { if (!value) {
if (this._rfbConnectionState === 'connected' && this._transitConnectionState == this.TransitConnectionStates.Udp) { if (this._rfbConnectionState === 'connected' && (this._transitConnectionState !== this.TransitConnectionStates.Tcp)) {
this._sendUdpDowngrade(); this._sendUdpDowngrade();
} }
} else { } else {
if (this._rfbConnectionState === 'connected' && (this._transitConnectionState == this.TransitConnectionStates.Tcp)) { if (this._rfbConnectionState === 'connected' && (this._transitConnectionState !== this.TransitConnectionStates.Udp)) {
this._sendUdpUpgrade(); this._sendUdpUpgrade();
} }
} }
@ -948,7 +950,7 @@ export default class RFB extends EventTargetMixin {
// ===== PRIVATE METHODS ===== // ===== PRIVATE METHODS =====
_changeTransitConnectionState(value) { _changeTransitConnectionState(value) {
Log.Debug("Transit state change from " + this._transitConnectionState.toString() + ' to ' + value.toString()); Log.Info("Transit state change from " + this._transitConnectionState.toString() + ' to ' + value.toString());
this._transitConnectionState = value; this._transitConnectionState = value;
} }
@ -1087,6 +1089,10 @@ export default class RFB extends EventTargetMixin {
(u8[15] << 24), 10); (u8[15] << 24), 10);
// TODO: check the hash. It's the low 32 bits of XXH64, seed 0 // TODO: check the hash. It's the low 32 bits of XXH64, seed 0
if (me._transitConnectionState !== me.TransitConnectionStates.Udp) {
me._changeTransitConnectionState(me.TransitConnectionStates.Udp);
}
if (pieces == 1) { // Handle it immediately if (pieces == 1) { // Handle it immediately
me._handleUdpRect(u8.slice(16)); me._handleUdpRect(u8.slice(16));
} else { // Insert into wait array } else { // Insert into wait array
@ -3134,7 +3140,6 @@ export default class RFB extends EventTargetMixin {
var candidate = new RTCIceCandidate(response.candidate); var candidate = new RTCIceCandidate(response.candidate);
peer.addIceCandidate(candidate).then(function() { peer.addIceCandidate(candidate).then(function() {
Log.Debug("success in addicecandidate"); Log.Debug("success in addicecandidate");
this._changeTransitConnectionState(this.TransitConnectionStates.Udp);
}.bind(this)).catch(function(err) { }.bind(this)).catch(function(err) {
Log.Error("Failure in addIceCandidate", err); Log.Error("Failure in addIceCandidate", err);
this._changeTransitConnectionState(this.TransitConnectionStates.Failure) this._changeTransitConnectionState(this.TransitConnectionStates.Failure)
@ -3184,9 +3189,7 @@ export default class RFB extends EventTargetMixin {
this._FBU.encoding = null; this._FBU.encoding = null;
} }
if (document.visibilityState !== "hidden") {
this._display.flip(); this._display.flip();
}
return true; // We finished this FBU return true; // We finished this FBU
} }
@ -3515,6 +3518,8 @@ export default class RFB extends EventTargetMixin {
Log.Warn("UDP connection failures exceeded limit, remaining on TCP transit.") Log.Warn("UDP connection failures exceeded limit, remaining on TCP transit.")
} }
} }
} else if (this._transitConnectionState == this.TransitConnectionStates.Downgrading) {
this._changeTransitConnectionState(this.TransitConnectionStates.Tcp);
} }
return decoder.decodeRect(this._FBU.x, this._FBU.y, return decoder.decodeRect(this._FBU.x, this._FBU.y,
this._FBU.width, this._FBU.height, this._FBU.width, this._FBU.height,

4
tests/README.md
View File

@ -23,6 +23,10 @@ mv /home/ubuntu/record.bin.8 /usr/share/kasmvnc/www/recordings
Place recordings on the KasmVNC server in the /usr/share/kasmvnc/www/recordings directory, you may need to create this directory. Then navigate to https://server-ip:8444/tests/vnc_playback.html?data=record.bin.8 where record.bin.8 is the name of the playback file you placed in the recordings directory. Place recordings on the KasmVNC server in the /usr/share/kasmvnc/www/recordings directory, you may need to create this directory. Then navigate to https://server-ip:8444/tests/vnc_playback.html?data=record.bin.8 where record.bin.8 is the name of the playback file you placed in the recordings directory.
## Pre-Test Modifications
Before running performance testing using recording playback, you need to run noVNC from source, rather than the 'compiled' webpack. See the docs at docs/DEVELOP.md for running noVNC from source.
## Kasm Provided Recordings ## Kasm Provided Recordings
The following recordings are used by Kasm Technologies to provide repeatable performance statisitics using different rendering settings. The following recordings are used by Kasm Technologies to provide repeatable performance statisitics using different rendering settings.