Merge branch 'feature/KASM-982_multi_monitor' into feature/KASM-5078_multi_monitor_display_ui

core/base64.js

@@ -58,7 +58,7 @@ export default {
 
         /* Every four characters is 3 resulting numbers */
         const resultLength = (dataLength >> 2) * 3 + Math.floor((dataLength % 4) / 1.5);
-        const result = new Array(resultLength);
+        const result = new Uint8Array(resultLength);
 
         // Convert one by one.
 

core/display.js

@@ -12,6 +12,7 @@ import Base64 from "./base64.js";
 import { toSigned32bit } from './util/int.js';
 import { isWindows } from './util/browser.js';
 import { uuidv4 } from './util/strings.js';
+import base64 from './base64.js';
 
 export default class Display {
     constructor(target, isPrimaryDisplay) {
@@ -31,6 +32,7 @@ export default class Display {
         this._asyncFrameQueue = [];
         this._maxAsyncFrameQueue = 3;
         this._clearAsyncQueue();
+        this._syncFrameQueue = [];
 
         this._flushing = false;
 
@@ -110,12 +112,11 @@ export default class Display {
 
         this.onflush = () => {  }; // A flush request has finished
 
-        // Use requestAnimationFrame to write to canvas, to match display refresh rate
-        this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
-
         if (!this._isPrimaryDisplay) {
             this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`);
             this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this));
+        } else {
+            this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
         }
 
         Log.Debug("<< Display.constructor");
@@ -482,10 +483,11 @@ export default class Display {
     */
     flush(onflush_message=true) {
         //force oldest frame to render
+        //window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
         this._asyncFrameComplete(0, true);
 
         if (onflush_message)
-            this._flushing = true;
+            this.onflush();
     }
     
     /*
@@ -501,7 +503,6 @@ export default class Display {
     */
     dispose() {
         clearInterval(this._frameStatsInterval);
-        cancelAnimationFrame(this._animationFrameID);
         this.clear();
     }
 
@@ -710,45 +711,24 @@ export default class Display {
                     //overwrite screen locations when received on the secondary display
                     rect.screenLocations = [ rect.screenLocations[event.data.screenLocationIndex] ]
                     rect.screenLocations[0].screenIndex = 0;
-                    let pos = rect.screenLocations[0];
-                    //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`)
                     switch (rect.type) {
-                        case 'copy':
-                            //this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true);
-                            this._asyncRenderQPush(rect);
-                            break;
-                        case 'fill':
-                            this._asyncRenderQPush(rect);
-                            //this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true);
-                            break;
-                        case 'blit':
-                            this._asyncRenderQPush(rect);
-                            //this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true);
-                            break;
-                        case 'blitQ':
-                            
-                            this._asyncRenderQPush(rect);
-                            //this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true);
-                            break;
                         case 'img':
                         case '_img':
                             rect.img = new Image();
                             rect.img.src = rect.src;
                             rect.type = 'img';
-                            this._asyncRenderQPush(rect);
                             break;
                         case 'transparent':
                             let imageBmpPromise = createImageBitmap(rect.arr);
                             imageBmpPromise.then(function(rect, img) {
                                 rect.img.complete = true;
                             }).bind(this, rect);
-                            this._asyncRenderQPush(rect);
                             break;
                     }
+                    this._syncFrameQueue.push(rect);
                     break;
                 case 'frameComplete':
-                        this.flip(event.data.frameId, event.data.rectCnt);
+                        window.requestAnimationFrame( () => { this._pushSyncRects(); });
-
                         break;
                 case 'registered':
                         if (!this._isPrimaryDisplay) {
@@ -760,6 +740,59 @@ export default class Display {
         }
     }
 
+    _pushSyncRects() {
+        whileLoop:
+        while (this._syncFrameQueue.length > 0) {
+            const a = this._syncFrameQueue[0];
+            const pos = a.screenLocations[0];
+            switch (a.type) {
+                case 'copy':
+                    this.copyImage(pos.oldX, pos.oldY, pos.x, pos.y, a.width, a.height, a.frame_id, true);
+                    break;
+                case 'fill':
+                    this.fillRect(pos.x, pos.y, a.width, a.height, a.color, a.frame_id, true);
+                    break;
+                case 'blit':
+                    this.blitImage(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true);
+                    break;
+                case 'blitQ':
+                    this.blitQoi(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true);
+                    break;
+                case 'img':
+                    if (a.img.complete) {
+                        this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
+                    } else {
+                        if (this._syncFrameQueue.length > 1000) {
+                            this._syncFrameQueue.shift();
+                            this._droppedRects++;
+                        } else {
+                            break whileLoop;
+                        }
+                    }
+                    break;
+                case 'transparent':
+                    if (a.img.complete) {
+                        this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
+                    } else {
+                        if (this._syncFrameQueue.length > 1000) {
+                            this._syncFrameQueue.shift();
+                            this._droppedRects++;
+                        } else {
+                            break whileLoop;
+                        }
+                    }
+                    break;
+                default:
+                    Log.Warn(`Unknown rect type: ${rect}`);
+            }
+            this._syncFrameQueue.shift();
+        }
+
+        if (this._syncFrameQueue.length > 0) {
+            window.requestAnimationFrame( () => { this._pushSyncRects(); });
+        }
+    }
+
     /*
     Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding
     */
@@ -796,7 +829,7 @@ export default class Display {
                 }  
             }
 
-            if (this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) {
+            if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) {
                 //frame is complete
                 this._asyncFrameComplete(frameIx);
             }
@@ -847,10 +880,15 @@ export default class Display {
                 }
             }
             while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) {   
-                if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
+                if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') {
+                    if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
                         this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip';
                         this._droppedRects++;
+                    } else {
+                        Log.Warn(`Oh snap, an image rect without an image: ${this._asyncFrameQueue[frameIx][2][currentFrameRectIx]}`)
                     }
+                }
+
                 currentFrameRectIx++;
             }
         } else {
@@ -868,6 +906,8 @@ export default class Display {
         }
         this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
         this._asyncFrameQueue[frameIx][3] = true;
+
+        //window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
     }
 
     /*
@@ -916,6 +956,7 @@ export default class Display {
                         if (a.img) {
                             a.img = null;
                         }
+
                         if (a.type !== 'flip') {
                             secondaryScreenRects++;
                             this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI });