jcarr
/
noVNC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bugfix/kasm 5417 watermark multi mon (#85) 

* KASM-5417 fix bug watermrk multi-display
* Force the zlib expected size to an int, it was unexpectedly float
* KASM-5427 transparent rects not drawn on primary display
* add double buffer when transparent rects are used
* refactor of transparent rects
* Use a more descriptive zlib error
* account for dropped transparent rects
* fix bug with multi-monitor with overlay
This commit is contained in:
Matt McClaskey 2024-01-17 10:44:20 -05:00 committed by GitHub
parent 4dac080460
commit d590bb931a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 200 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
core/decoders/tight.js
View File

@ -11,6 +11,7 @@
import * as Log from '../util/logging.js';
import Inflator from "../inflator.js";
import { hashUInt8Array } from '../util/int.js';
export default class TightDecoder {
constructor(display) {
@ -21,6 +22,8 @@ export default class TightDecoder {
this._len = 0;
this._enableQOI = false;
this._displayGlobal = display;
this._lastTransparentRectHash = '';
this._lastTransparentRectInfo = '';
this._zlibs = [];
for (let i = 0; i < 4; i++) {
@ -183,44 +186,52 @@ export default class TightDecoder {
return true;
}
// intensity tinted
_itRect(x, y, width, height, sock, display, depth, frame_id) {
let data = this._readData(sock);
if (data === null) {
return false;
}
const r = data[0];
const g = data[1];
const b = data[2];
const a = data[3];
//filter out consecutive redundant data
let h = hashUInt8Array(data);
let info = `${x}.${y}.${width}.${height}`
if (!(h === this._lastTransparentRectHash && info === this._lastTransparentRectInfo)) {
const r = data[0];
const g = data[1];
const b = data[2];
const a = data[3];
const uncompressedSize = width * height / 2 + 1;
const uncompressedSize = Math.floor(width * height / 2 + 1);
this._itzlib.reset();
this._itzlib.setInput(data.slice(4));
data = this._itzlib.inflate(uncompressedSize);
this._itzlib.setInput(null);
this._itzlib.reset();
this._itzlib.setInput(data.slice(4));
data = this._itzlib.inflate(uncompressedSize);
this._itzlib.setInput(null);
// unpack
let rgba = new Uint8Array(width * height * 4 + 4);
for (let i = 0, d = 0; i < uncompressedSize; i++, d += 8) {
let p = data[i];
// unpack
let rgba = new Uint8Array(width * height * 4 + 4);
for (let i = 0, d = 0; i < uncompressedSize; i++, d += 8) {
let p = data[i];
rgba[d + 0] = r;
rgba[d + 1] = g;
rgba[d + 2] = b;
rgba[d + 3] = a * ((p & 15) << 4) / 255;
rgba[d + 0] = r;
rgba[d + 1] = g;
rgba[d + 2] = b;
rgba[d + 3] = a * ((p & 15) << 4) / 255;
rgba[d + 4] = r;
rgba[d + 5] = g;
rgba[d + 6] = b;
rgba[d + 7] = a * (p & 240) / 255;
rgba[d + 4] = r;
rgba[d + 5] = g;
rgba[d + 6] = b;
rgba[d + 7] = a * (p & 240) / 255;
}
let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height);
display.transparentRect(x, y, width, height, img, frame_id, h);
this._lastTransparentRectHash = h;
this._lastTransparentRectInfo = info;
} else {
display.dummyRect(x, y, width, height, frame_id);
}
let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height);
display.transparentRect(x, y, width, height, img, frame_id);
return true;
}

235
core/display.js
View File

@ -32,6 +32,9 @@ export default class Display {
this._maxAsyncFrameQueue = 3;
this._clearAsyncQueue();
this._syncFrameQueue = [];
this._transparentOverlayImg = null;
this._transparentOverlayRect = null;
this._lastTransparentRectId = "";
this._flushing = false;
@ -75,7 +78,7 @@ export default class Display {
if (delta > 0) {
this._fps = (this._flipCnt / (delta / 1000)).toFixed(2);
}
Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect);
Log.Debug('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Forced Frames: ' + this._forcedFrameCnt + ' Missing Flips: ' + this._missingFlipRect + ' Late Flips: ' + this._lateFlipRect);
this._flipCnt = 0;
this._lastFlip = Date.now();
}.bind(this), 5000);
@ -107,6 +110,12 @@ export default class Display {
channel: null
}];
//optional offscreen canvas
this._enableCanvasBuffer = false;
this._backbuffer = document.createElement('canvas');
this._drawCtx = this._backbuffer.getContext('2d');
this._damageBounds = { left: 0, top: 0, right: this._backbuffer.width, bottom: this._backbuffer.height };
// ===== EVENT HANDLERS =====
this.onflush = () => { }; // A flush request has finished
@ -123,6 +132,29 @@ export default class Display {
// ===== PROPERTIES =====
get enableCanvasBuffer() { return this._enableCanvasBuffer; }
set enableCanvasBuffer(value) {
if (value === this._enableCanvasBuffer) { return; }
this._enableCanvasBuffer = value;
if (value && this._target)
{
//copy current visible canvas to backbuffer
let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height);
this._drawCtx.putImageData(saveImg, 0, 0);
if (this._transparentOverlayImg) {
this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
}
} else if (!value && this._target) {
//copy backbuffer to canvas to clear any overlays
let saveImg = this._targetCtx.getImageData(0, 0, this._target.width, this._target.height);
this._drawCtx.putImageData(saveImg, 0, 0);
}
}
get screens() { return this._screens; }
get screenID() { return this._screenID; }
get screenIndex() {
@ -419,14 +451,23 @@ export default class Display {
}
const vp = this._screens[0];
if (vp.serverWidth !== width || vp.serverHeight !== height) {
const canvas = this._target;
if (canvas.width !== width || canvas.height !== height) {
let saveImg = null;
if (canvas.width > 0 && canvas.height > 0) {
saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height);
}
vp.serverWidth = width;
vp.serverHeight = height;
const canvas = this._target;
canvas.width = width;
canvas.height = height;
if (saveImg) {
this._targetCtx.putImageData(saveImg, 0, 0);
}
// The position might need to be updated if we've grown
this.viewportChangePos(0, 0);
@ -455,32 +496,36 @@ export default class Display {
this._fbWidth = width;
this._fbHeight = height;
const canvas = this._target;
let canvas = this._backbuffer;
if (canvas == undefined) { return; }
if (this._screens.length > 0) {
width = this._screens[0].serverWidth;
height = this._screens[0].serverHeight;
}
if (canvas.width !== width || canvas.height !== height) {
if (canvas.width !== width || canvas.height !== height) {
// We have to save the canvas data since changing the size will clear it
let saveImg = null;
if (canvas.width > 0 && canvas.height > 0) {
saveImg = this._targetCtx.getImageData(0, 0, canvas.width, canvas.height);
saveImg = this._drawCtx.getImageData(0, 0, canvas.width, canvas.height);
}
if (canvas.width !== width) {
canvas.width = width;
}
if (canvas.height !== height) {
canvas.height = height;
}
if (saveImg) {
this._targetCtx.putImageData(saveImg, 0, 0);
this._drawCtx.putImageData(saveImg, 0, 0);
}
}
// Readjust the viewport as it may be incorrectly sized
// and positioned
const vp = this._screens[0];
@ -556,7 +601,11 @@ export default class Display {
this._asyncRenderQPush(rect);
} else {
this._setFillColor(color);
this._targetCtx.fillRect(x, y, width, height);
if (this._enableCanvasBuffer) {
this._drawCtx.fillRect(x, y, width, height);
} else {
this._targetCtx.fillRect(x, y, width, height);
}
}
}
@ -575,6 +624,9 @@ export default class Display {
this._processRectScreens(rect);
this._asyncRenderQPush(rect);
} else {
let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx);
let sourceCvs = ((this._enableCanvasBuffer) ? this._backbuffer : this._target);
// Due to this bug among others [1] we need to disable the image-smoothing to
// avoid getting a blur effect when copying data.
//
@ -582,12 +634,12 @@ export default class Display {
//
// We need to set these every time since all properties are reset
// when the the size is changed
this._targetCtx.mozImageSmoothingEnabled = false;
this._targetCtx.webkitImageSmoothingEnabled = false;
this._targetCtx.msImageSmoothingEnabled = false;
this._targetCtx.imageSmoothingEnabled = false;
targetCtx.mozImageSmoothingEnabled = false;
targetCtx.webkitImageSmoothingEnabled = false;
targetCtx.msImageSmoothingEnabled = false;
targetCtx.imageSmoothingEnabled = false;
this._targetCtx.drawImage(this._target,
targetCtx.drawImage(sourceCvs,
oldX, oldY, w, h,
newX, newY, w, h);
}
@ -625,7 +677,7 @@ export default class Display {
this._asyncRenderQPush(rect);
}
transparentRect(x, y, width, height, img, frame_id) {
transparentRect(x, y, width, height, img, frame_id, hashId) {
/* The internal logic cannot handle empty images, so bail early */
if ((width === 0) || (height === 0)) {
return;
@ -638,21 +690,35 @@ export default class Display {
'y': y,
'width': width,
'height': height,
'frame_id': frame_id
'frame_id': frame_id,
'arr': img,
'hash_id': hashId
}
this._processRectScreens(rect);
if (rect.inPrimary) {
let imageBmpPromise = createImageBitmap(img);
imageBmpPromise.then( function(img) {
rect.img = img;
rect.img.complete = true;
}.bind(rect) );
}
if (rect.inSecondary) {
rect.arr = img;
imageBmpPromise.then( function(bitmap) {
this._transparentOverlayImg = bitmap;
this.enableCanvasBuffer = true;
}.bind(this) );
}
this._transparentOverlayRect = rect;
this._asyncRenderQPush(rect);
}
dummyRect(x, y, width, height, frame_id) {
let rect = {
'type': 'dummy',
'img': null,
'x': x,
'y': y,
'width': width,
'height': height,
'frame_id': frame_id
}
this._processRectScreens(rect);
this._asyncRenderQPush(rect);
}
@ -680,7 +746,13 @@ export default class Display {
arr.byteOffset + offset,
width * height * 4);
let img = new ImageData(data, width, height);
this._targetCtx.putImageData(img, x, y);
if (this._enableCanvasBuffer) {
this._drawCtx.putImageData(img, x, y);
} else {
this._targetCtx.putImageData(img, x, y);
}
}
}
@ -698,16 +770,21 @@ export default class Display {
this._processRectScreens(rect);
this._asyncRenderQPush(rect);
} else {
this._targetCtx.putImageData(arr, x, y);
if (this._enableCanvasBuffer) {
this._drawCtx.putImageData(arr, x, y);
} else {
this._targetCtx.putImageData(arr, x, y);
}
}
}
drawImage(img, x, y, w, h) {
drawImage(img, x, y, w, h, overlay=false) {
try {
let targetCtx = ((this._enableCanvasBuffer && !overlay) ? this._drawCtx : this._targetCtx);
if (img.width != w || img.height != h) {
this._targetCtx.drawImage(img, x, y, w, h);
targetCtx.drawImage(img, x, y, w, h);
} else {
this._targetCtx.drawImage(img, x, y);
targetCtx.drawImage(img, x, y);
}
} catch (error) {
Log.Error('Invalid image recieved.'); //KASM-2090
@ -737,9 +814,13 @@ export default class Display {
// ===== PRIVATE METHODS =====
_writeCtxBuffer() {
//TODO: KASM-5450 Damage tracking with transparent rect overlay support
this._targetCtx.drawImage(this._backbuffer, 0, 0);
}
_handleSecondaryDisplayMessage(event) {
if (!this._isPrimaryDisplay && event.data) {
switch (event.data.eventType) {
case 'rect':
let rect = event.data.rect;
@ -755,9 +836,11 @@ export default class Display {
break;
case 'transparent':
let imageBmpPromise = createImageBitmap(rect.arr);
imageBmpPromise.then(function(rect, img) {
rect.img.complete = true;
}).bind(this, rect);
imageBmpPromise.then( function(img) {
this._transparentOverlayImg = img;
this.enableCanvasBuffer = true;
}.bind(this) );
this._transparentOverlayRect = rect;
break;
}
this._syncFrameQueue.push(rect);
@ -768,7 +851,10 @@ export default class Display {
case 'registered':
if (!this._isPrimaryDisplay) {
this._screens[0].screenIndex = event.data.screenIndex;
Log.Error(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`);
Log.Info(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`);
if (this._screens.length > 0) {
this.resize(this._screens[0].serverWidth, this._screens[0].serverHeight);
}
}
break;
}
@ -776,6 +862,7 @@ export default class Display {
}
_pushSyncRects() {
let drawRectCnt = 0;
whileLoop:
while (this._syncFrameQueue.length > 0) {
const a = this._syncFrameQueue[0];
@ -805,24 +892,21 @@ export default class Display {
}
}
break;
case 'transparent':
if (a.img.complete) {
this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
} else {
if (this._syncFrameQueue.length > 1000) {
this._syncFrameQueue.shift();
this._droppedRects++;
} else {
break whileLoop;
}
}
break;
default:
Log.Warn(`Unknown rect type: ${rect}`);
this._syncFrameQueue.shift();
continue;
}
drawRectCnt++;
this._syncFrameQueue.shift();
}
if (this._enableCanvasBuffer && drawRectCnt > 0) {
this._writeCtxBuffer();
if (this._transparentOverlayImg) {
this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
}
}
if (this._syncFrameQueue.length > 0) {
window.requestAnimationFrame( () => { this._pushSyncRects(); });
}
@ -963,8 +1047,6 @@ export default class Display {
this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.addEventListener('load', () => { this._asyncFrameComplete(frameIx); });
this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
return;
} else if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'transparent' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img) {
return;
}
currentFrameRectIx++;
@ -991,8 +1073,8 @@ export default class Display {
this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ]);
}
let transparent_rects = [];
let secondaryScreenRects = 0;
let primaryScreenRects = 0;
//render the selected frame
for (let i = 0; i < frame.length; i++) {
@ -1018,38 +1100,41 @@ export default class Display {
case 'img':
this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height);
break;
case 'transparent':
transparent_rects.push(a);
break;
default:
continue;
}
primaryScreenRects++;
} else {
if (a.img) {
a.img = null;
}
if (a.type !== 'flip') {
secondaryScreenRects++;
this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });
switch (a.type) {
case 'dummy':
case 'transparent':
case 'flip':
break;
default:
secondaryScreenRects++;
a.img = null;
this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });
}
}
}
}
//rects with transparency get applied last
for (let i = 0; i < transparent_rects.length; i++) {
const a = transparent_rects[i];
let screenIndexes = this._getRectScreenIndexes(a);
if (this._enableCanvasBuffer) {
for (let sI = 0; sI < screenLocations.length; sI++) {
let screenLocation = a.screenLocations[sI];
if (sI == 0) {
if (a.img) {
this.drawImage(a.img, a.x, a.y, a.width, a.height);
}
} else {
secondaryScreenRects++;
this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });
if (primaryScreenRects > 0) {
this._writeCtxBuffer();
}
if (this._transparentOverlayImg) {
if (primaryScreenRects > 0) {
this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
}
if (secondaryScreenRects > 0 && this._lastTransparentRectId !== this._transparentOverlayRect.hash_id) {
for (let sI = 1; sI < this._transparentOverlayRect.screenLocations.length; sI++) {
this._screens[this._transparentOverlayRect.screenLocations[sI].screenIndex].channel.postMessage({ eventType: 'rect', rect: this._transparentOverlayRect, screenLocationIndex: sI });
}
}
this._lastTransparentRectId = this._transparentOverlayRect.hash_id;
}
}
@ -1065,6 +1150,11 @@ export default class Display {
this._flushing = false;
this.onflush();
}
// if there is more data in queue, then keep checking
if (this._asyncFrameQueue[0][2].length > 0) {
window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
}
} else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] == this._asyncFrameQueue[0][2].length) {
//how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn
this._asyncFrameQueue[0][5] += 1;
@ -1141,8 +1231,9 @@ export default class Display {
_setFillColor(color) {
const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')';
let targetCtx = ((this._enableCanvasBuffer) ? this._drawCtx : this._targetCtx);
if (newStyle !== this._prevDrawStyle) {
this._targetCtx.fillStyle = newStyle;
targetCtx.fillStyle = newStyle;
this._prevDrawStyle = newStyle;
}
}

2
core/inflator.js
View File

@ -54,7 +54,7 @@ export default class Inflate {
}
if (this.strm.next_out != expected) {
throw new Error("Incomplete zlib block");
throw new Error("Incomplete zlib block, got " + this.strm.next_out + " expected " + expected);
}
return new Uint8Array(this.strm.output.buffer, 0, this.strm.next_out);

3
core/rfb.js
View File

@ -217,7 +217,6 @@ export default class RFB extends EventTargetMixin {
this._gestureLastMagnitudeY = 0;
// Secondary Displays
this._secondaryDisplays = {};
this._supportsBroadcastChannel = (typeof BroadcastChannel !== "undefined");
if (this._supportsBroadcastChannel) {
this._controlChannel = new BroadcastChannel(this._connectionID);
@ -1750,8 +1749,6 @@ export default class RFB extends EventTargetMixin {
Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`);
break;
case 'reattach':
console.log('reattach message')
console.log(event.data)
this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth);
size = this._screenSize();
RFB.messages.setDesktopSize(this._sock, size, this._screenFlags);