From f243b54ac9154fdb1efd953520a45ebd3b405aae Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 06:02:33 -0400 Subject: [PATCH 1/3] fix bugs in async frame sync --- core/display.js | 56 ++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/core/display.js b/core/display.js index ab7b4b80..3fc9fa60 100644 --- a/core/display.js +++ b/core/display.js @@ -110,9 +110,6 @@ export default class Display { this.onflush = () => { }; // A flush request has finished - // Use requestAnimationFrame to write to canvas, to match display refresh rate - this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); - if (!this._isPrimaryDisplay) { this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`); this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this)); @@ -482,10 +479,10 @@ export default class Display { */ flush(onflush_message=true) { //force oldest frame to render - this._asyncFrameComplete(0, true); + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); if (onflush_message) - this._flushing = true; + this.onflush(); } /* @@ -501,7 +498,6 @@ export default class Display { */ dispose() { clearInterval(this._frameStatsInterval); - cancelAnimationFrame(this._animationFrameID); this.clear(); } @@ -714,40 +710,45 @@ export default class Display { //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`) switch (rect.type) { case 'copy': - //this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); - this._asyncRenderQPush(rect); + this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); + //this._asyncRenderQPush(rect); break; case 'fill': - this._asyncRenderQPush(rect); - //this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); break; case 'blit': - this._asyncRenderQPush(rect); - //this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'blitQ': - - this._asyncRenderQPush(rect); - //this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'img': case '_img': rect.img = new Image(); rect.img.src = rect.src; rect.type = 'img'; - this._asyncRenderQPush(rect); + //this._asyncRenderQPush(rect); + if (!rect.img.complete) { + rect.img.addEventListener('load', (rect) => { + this.drawImage(rect.img, rect.x, rect.y, rect.width, rect.height); + }); + } break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); imageBmpPromise.then(function(rect, img) { - rect.img.complete = true; + //rect.img.complete = true; + this.drawImage(img, rect.x, rect.y, rect.width, rect.height); }).bind(this, rect); - this._asyncRenderQPush(rect); + //this._asyncRenderQPush(rect); break; } break; case 'frameComplete': - this.flip(event.data.frameId, event.data.rectCnt); + //this.flip(event.data.frameId, event.data.rectCnt); break; case 'registered': @@ -847,10 +848,15 @@ export default class Display { } } while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { - if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { - this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; - this._droppedRects++; + if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') { + if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { + this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; + this._droppedRects++; + } else { + Log.Warn(`Oh snap, an image rect without an image: ${this._asyncFrameQueue[frameIx][2][currentFrameRectIx]}`) + } } + currentFrameRectIx++; } } else { @@ -868,6 +874,8 @@ export default class Display { } this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; this._asyncFrameQueue[frameIx][3] = true; + + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } /* @@ -962,10 +970,6 @@ export default class Display { this._pushAsyncFrame(true); } } - - if (!force) { - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); - } } _processRectScreens(rect) { From 56a2e2dec94ac422d9f73be9b4bfd671ee8bfcf2 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 07:51:50 -0400 Subject: [PATCH 2/3] add syncronous frames for secondary display --- core/base64.js | 2 +- core/display.js | 88 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/core/base64.js b/core/base64.js index e4b6db79..f910ed26 100644 --- a/core/base64.js +++ b/core/base64.js @@ -58,7 +58,7 @@ export default { /* Every four characters is 3 resulting numbers */ const resultLength = (dataLength >> 2) * 3 + Math.floor((dataLength % 4) / 1.5); - const result = new Array(resultLength); + const result = new Uint8Array(resultLength); // Convert one by one. diff --git a/core/display.js b/core/display.js index 3fc9fa60..685192e1 100644 --- a/core/display.js +++ b/core/display.js @@ -12,6 +12,7 @@ import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; import { uuidv4 } from './util/strings.js'; +import base64 from './base64.js'; export default class Display { constructor(target, isPrimaryDisplay) { @@ -31,6 +32,7 @@ export default class Display { this._asyncFrameQueue = []; this._maxAsyncFrameQueue = 3; this._clearAsyncQueue(); + this._syncFrameQueue = []; this._flushing = false; @@ -706,50 +708,24 @@ export default class Display { //overwrite screen locations when received on the secondary display rect.screenLocations = [ rect.screenLocations[event.data.screenLocationIndex] ] rect.screenLocations[0].screenIndex = 0; - let pos = rect.screenLocations[0]; - //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`) switch (rect.type) { - case 'copy': - this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); - //this._asyncRenderQPush(rect); - break; - case 'fill': - //this._asyncRenderQPush(rect); - this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); - break; - case 'blit': - //this._asyncRenderQPush(rect); - this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); - break; - case 'blitQ': - //this._asyncRenderQPush(rect); - this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); - break; case 'img': case '_img': rect.img = new Image(); rect.img.src = rect.src; rect.type = 'img'; - //this._asyncRenderQPush(rect); - if (!rect.img.complete) { - rect.img.addEventListener('load', (rect) => { - this.drawImage(rect.img, rect.x, rect.y, rect.width, rect.height); - }); - } break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); imageBmpPromise.then(function(rect, img) { - //rect.img.complete = true; - this.drawImage(img, rect.x, rect.y, rect.width, rect.height); + rect.img.complete = true; }).bind(this, rect); - //this._asyncRenderQPush(rect); break; } + this._syncFrameQueue.push(rect); break; case 'frameComplete': - //this.flip(event.data.frameId, event.data.rectCnt); - + window.requestAnimationFrame( () => { this._pushSyncRects(); }); break; case 'registered': if (!this._isPrimaryDisplay) { @@ -761,6 +737,59 @@ export default class Display { } } + _pushSyncRects() { + whileLoop: + while (this._syncFrameQueue.length > 0) { + const a = this._syncFrameQueue[0]; + const pos = a.screenLocations[0]; + switch (a.type) { + case 'copy': + this.copyImage(pos.oldX, pos.oldY, pos.x, pos.y, a.width, a.height, a.frame_id, true); + break; + case 'fill': + this.fillRect(pos.x, pos.y, a.width, a.height, a.color, a.frame_id, true); + break; + case 'blit': + this.blitImage(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'blitQ': + this.blitQoi(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'img': + if (a.img.complete) { + this.drawImage(a.img, pos.x, pos.y, a.width, a.height); + } else { + if (this._syncFrameQueue.length > 1000) { + this._syncFrameQueue.shift(); + this._droppedRects++; + } else { + break whileLoop; + } + } + break; + case 'transparent': + if (a.img.complete) { + this.drawImage(a.img, pos.x, pos.y, a.width, a.height); + } else { + if (this._syncFrameQueue.length > 1000) { + this._syncFrameQueue.shift(); + this._droppedRects++; + } else { + break whileLoop; + } + } + break; + default: + Log.Warn(`Unknown rect type: ${rect}`); + } + this._syncFrameQueue.shift(); + } + + if (this._syncFrameQueue.length > 0) { + window.requestAnimationFrame( () => { this._pushSyncRects(); }); + } + } + /* Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding */ @@ -924,6 +953,7 @@ export default class Display { if (a.img) { a.img = null; } + if (a.type !== 'flip') { secondaryScreenRects++; this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); From 70faacff85bfc2c0c58d77a7cc29ac0abd0751ec Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 09:19:00 -0400 Subject: [PATCH 3/3] smashed a frame dropping bug --- core/display.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/display.js b/core/display.js index 685192e1..d19e05d6 100644 --- a/core/display.js +++ b/core/display.js @@ -115,6 +115,8 @@ export default class Display { if (!this._isPrimaryDisplay) { this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`); this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this)); + } else { + this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } Log.Debug("<< Display.constructor"); @@ -481,7 +483,8 @@ export default class Display { */ flush(onflush_message=true) { //force oldest frame to render - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + //window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + this._asyncFrameComplete(0, true); if (onflush_message) this.onflush(); @@ -826,7 +829,7 @@ export default class Display { } } - if (this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { + if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { //frame is complete this._asyncFrameComplete(frameIx); } @@ -904,7 +907,7 @@ export default class Display { this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; this._asyncFrameQueue[frameIx][3] = true; - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + //window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } /* @@ -1000,6 +1003,10 @@ export default class Display { this._pushAsyncFrame(true); } } + + if (!force) { + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + } } _processRectScreens(rect) {