Transparent Rect Type (#58)

* implement new transparent rect encoding
* implement a max frame hold time
---------

Co-authored-by: Lauri Kasanen <cand@gmx.com>
Co-authored-by: mattmcclaskey <matt@kasmweb.com>

core/decoders/tight.js

@@ -26,6 +26,7 @@ export default class TightDecoder {
         for (let i = 0; i < 4; i++) {
             this._zlibs[i] = new Inflator();
         }
+        this._itzlib = new Inflator();
     }
 
     // ===== PROPERTIES =====
@@ -86,6 +87,9 @@ export default class TightDecoder {
         } else if (this._ctl === 0x0C) {
             ret = this._qoiRect(x, y, width, height,
                                 sock, display, depth, frame_id);
+        } else if (this._ctl === 0x0D) {
+            ret = this._itRect(x, y, width, height,
+                                sock, display, depth, frame_id);
         } else {
             throw new Error("Illegal tight compression received (ctl: " +
                                    this._ctl + ")");
@@ -179,6 +183,47 @@ export default class TightDecoder {
         return true;
     }
 
+    // intensity tinted
+    _itRect(x, y, width, height, sock, display, depth, frame_id) {
+        let data = this._readData(sock);
+        if (data === null) {
+            return false;
+        }
+
+        const r = data[0];
+        const g = data[1];
+        const b = data[2];
+        const a = data[3];
+
+        const uncompressedSize = width * height / 2 + 1;
+
+        this._itzlib.reset();
+        this._itzlib.setInput(data.slice(4));
+        data = this._itzlib.inflate(uncompressedSize);
+        this._itzlib.setInput(null);
+
+        // unpack
+        let rgba = new Uint8Array(width * height * 4 + 4);
+        for (let i = 0, d = 0; i < uncompressedSize; i++, d += 8) {
+            let p = data[i];
+
+            rgba[d + 0] = r;
+            rgba[d + 1] = g;
+            rgba[d + 2] = b;
+            rgba[d + 3] = a * ((p & 15) << 4) / 255;
+
+            rgba[d + 4] = r;
+            rgba[d + 5] = g;
+            rgba[d + 6] = b;
+            rgba[d + 7] = a * (p & 240) / 255;
+        }
+
+        let img = new ImageData(new Uint8ClampedArray(rgba.buffer, 0, width * height * 4), width, height);
+        display.transparentRect(x, y, width, height, img, frame_id);
+
+        return true;
+    }
+
     _pngRect(x, y, width, height, sock, display, depth, frame_id) {
         throw new Error("PNG received in standard Tight rect");
     }

core/display.js

@@ -25,6 +25,7 @@ export default class Display {
             2 - Array of Rect objects
             3 - bool, is the frame complete
             4 - int, index of current rect (post-processing)
+            5 - int, number of times requestAnimationFrame called _pushAsyncFrame and the frame had all rects, however, the frame was not marked complete
         */
         this._asyncFrameQueue = [];
         this._maxAsyncFrameQueue = 3;
@@ -377,6 +378,31 @@ export default class Display {
         });
     }
 
+    transparentRect(x, y, width, height, img, frame_id) {
+        /* The internal logic cannot handle empty images, so bail early */
+        if ((width === 0) || (height === 0)) {
+            return;
+        }
+
+        var rect = {
+            'type': 'transparent',
+            'img': null,
+            'x': x,
+            'y': y,
+            'width': width,
+            'height': height,
+            'frame_id': frame_id
+        }
+
+        let imageBmpPromise = createImageBitmap(img);
+        imageBmpPromise.then( function(img) {
+            rect.img = img;
+            rect.img.complete = true;
+        }.bind(rect) );
+
+        this._asyncRenderQPush(rect);
+    }
+
     blitImage(x, y, width, height, arr, offset, frame_id, fromQueue) {
         if (!fromQueue) {
             // NB(directxman12): it's technically more performant here to use preallocated arrays,
@@ -504,7 +530,7 @@ export default class Display {
                 //frame is newer than any frame in the queue, drop old frames
                 this._asyncFrameQueue.shift();
                 let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0);
-                this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0 ]);
+                this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0, 0 ]);
                 this._droppedFrames++;
             }
         }
@@ -519,7 +545,7 @@ export default class Display {
 
         this._asyncFrameQueue = [];
         for (let i=0; i<this._maxAsyncFrameQueue; i++) {
-            this._asyncFrameQueue.push([ 0, 0, [], false, 0 ])
+            this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ])
         }
     }
 
@@ -552,7 +578,10 @@ export default class Display {
                     this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.addEventListener('load', () => { this._asyncFrameComplete(frameIx); });
                     this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
                     return;
+                } else if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'transparent' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img) {
+                    return;
                 }
+
                 currentFrameRectIx++;
             }
         }
@@ -564,12 +593,14 @@ export default class Display {
     Push the oldest frame in the buffer to the canvas if it is marked ready
     */
     _pushAsyncFrame(force=false) {
-        if (this._asyncFrameQueue[0][3]) {
+        if (this._asyncFrameQueue[0][3] || force) {
             let frame = this._asyncFrameQueue.shift()[2];
             if (this._asyncFrameQueue.length < this._maxAsyncFrameQueue) {
-                this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]);
+                this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ]);
             }
 
+            let transparent_rects = [];
+            
             //render the selected frame
             for (let i = 0; i < frame.length; i++) {
                 
@@ -590,14 +621,34 @@ export default class Display {
                     case 'img':
                         this.drawImage(a.img, a.x, a.y, a.width, a.height);
                         break;
+                    case 'transparent':
+                        transparent_rects.push(a);
+                        break;
                 }
             }
+
+            //rects with transparency get applied last
+            for (let i = 0; i < transparent_rects.length; i++) {
+                const a = transparent_rects[i];
+                
+                if (a.img) {
+                    this.drawImage(a.img, a.x, a.y, a.width, a.height);
+                }
+            }
+
             this._flipCnt += 1;
 
             if (this._flushing) {
                 this._flushing = false;
                 this.onflush();
             }
+        } else if (this._asyncFrameQueue[0][1] > 0 && this._asyncFrameQueue[0][1] == this._asyncFrameQueue[0][2].length) {
+            //how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn
+            this._asyncFrameQueue[0][5] += 1;
+            //force the frame to be drawn if it has been here too long
+            if (this._asyncFrameQueue[0][5] > 5) { 
+                this._pushAsyncFrame(true);
+            }
         }
 
         if (!force) {