Major refactor of display class, support for QOI (#43)

* Major refactor of display.js queue to assume async rect processing in order to accommodate threaded decode or udp

* QOI lossless decoder on worker threads using WASM

* rfb.js class to provide frame_id and rect counts per frame

Co-authored-by: ryan.kuba <ryan.kuba@kasmweb.com>
Co-authored-by: matt <matt@kasmweb.com>

app/ui.js

@@ -245,6 +245,7 @@ const UI = {
         UI.initSetting('enable_perf_stats', false);
         UI.initSetting('virtual_keyboard_visible', false);
         UI.initSetting('enable_ime', false);
+        UI.initSetting('enable_qoi', false);
         UI.initSetting('enable_webrtc', false);
         UI.toggleKeyboardControls();
 
@@ -558,6 +559,8 @@ const UI = {
         UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls);
         UI.addSettingChangeHandler('enable_ime');
         UI.addSettingChangeHandler('enable_ime', UI.toggleIMEMode);
+        UI.addSettingChangeHandler('enable_qoi');
+        UI.addSettingChangeHandler('enable_qoi', UI.toggleQOI);
         UI.addSettingChangeHandler('enable_webrtc');
         UI.addSettingChangeHandler('enable_webrtc', UI.toggleWebRTC);
     },
@@ -1399,6 +1402,7 @@ const UI = {
         UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless');
         UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime');
         UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless;
+        UI.rfb.enableQOI = UI.getSetting('enable_qoi');
         UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc');
         UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper();
 
@@ -1651,6 +1655,18 @@ const UI = {
                         UI.toggleIMEMode();
                     }
                     break;
+                case 'disable_qoi':
+                    if(UI.getSetting('enable_qoi')) {
+                        UI.forceSetting('enable_qoi', false, false);
+                    }
+                    UI.toggleQOI();
+                    break;
+                case 'enable_qoi':
+                    if(!UI.getSetting('enable_qoi')) {
+                        UI.forceSetting('enable_qoi', true, false);
+                    }
+                    UI.toggleQOI();
+                    break;
                 case 'enable_webrtc':
                     if (!UI.getSetting('enable_webrtc')) {
                         UI.forceSetting('enable_webrtc', true, false);
@@ -1961,6 +1977,8 @@ const UI = {
                 UI.enableSetting('video_out_time');
                 UI.showStatus("Refresh or reconnect to apply changes.");
                 return;
+            case 5: //extreme+lossless
+                UI.forceSetting('enable_qoi', true, false);
             case 4: //extreme
                 UI.forceSetting('dynamic_quality_min', 9);
                 UI.forceSetting('dynamic_quality_max', 9);
@@ -2024,6 +2042,12 @@ const UI = {
                 break;
         }
 
+        //force QOI off if mode is below extreme
+        if (present_mode !== 4 && UI.getSetting('enable_qoi')) {
+            UI.showStatus("Lossless QOI disabled when not in extreme quality mode.");
+            UI.forceSetting('enable_qoi', false, false);
+        }
+
         if (UI.rfb) {
             UI.rfb.qualityLevel = parseInt(UI.getSetting('quality'));
             UI.rfb.antiAliasing = parseInt(UI.getSetting('anti_aliasing'));
@@ -2041,6 +2065,7 @@ const UI = {
             UI.rfb.frameRate = parseInt(UI.getSetting('framerate'));
             UI.rfb.enableWebP = UI.getSetting('enable_webp');
             UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality'));
+            UI.rfb.enableQOI = UI.getSetting('enable_qoi');
 
             // Gracefully update settings server side
             UI.rfb.updateConnectionSettings();
@@ -2096,9 +2121,27 @@ const UI = {
             } else {
                 UI.rfb.enableWebRTC = false;
             }
+            UI.updateQuality();
         }
     },
 
+    toggleQOI() {
+      if(UI.rfb) {
+          if(UI.getSetting('enable_qoi')) {
+              UI.rfb.enableQOI = true;
+              if (!UI.rfb.enableQOI) {
+                UI.showStatus("Enabling QOI failed, browser may not be compatible with WASM and/or Workers.");
+                UI.forceSetting('enable_qoi', false, false);
+                return;
+              }
+              UI.forceSetting('video_quality', 4, false); //force into extreme quality mode
+          } else {
+              UI.rfb.enableQOI = false;
+          }
+          UI.updateQuality();
+      }
+    },
+
     showKeyboardControls() {
         document.getElementById('noVNC_keyboard_control').classList.add("is-visible");
     },

core/decoders/copyrect.js

@@ -8,7 +8,8 @@
  */
 
 export default class CopyRectDecoder {
-    decodeRect(x, y, width, height, sock, display, depth) {
+
+    decodeRect(x, y, width, height, sock, display, depth, frame_id) {
         if (sock.rQwait("COPYRECT", 4)) {
             return false;
         }
@@ -20,7 +21,7 @@ export default class CopyRectDecoder {
             return true;
         }
 
-        display.copyImage(deltaX, deltaY, x, y, width, height);
+        display.copyImage(deltaX, deltaY, x, y, width, height, frame_id);
 
         return true;
     }

core/decoders/hextile.js

@@ -16,7 +16,7 @@ export default class HextileDecoder {
         this._tileBuffer = new Uint8Array(16 * 16 * 4);
     }
 
-    decodeRect(x, y, width, height, sock, display, depth) {
+    decodeRect(x, y, width, height, sock, display, depth, frame_id) {
         if (this._tiles === 0) {
             this._tilesX = Math.ceil(width / 16);
             this._tilesY = Math.ceil(height / 16);
@@ -85,7 +85,7 @@ export default class HextileDecoder {
                     // Weird: ignore blanks are RAW
                     Log.Debug("     Ignoring blank after RAW");
                 } else {
-                    display.fillRect(tx, ty, tw, th, this._background);
+                    display.fillRect(tx, ty, tw, th, this._background, frame_id);
                 }
             } else if (subencoding & 0x01) {  // Raw
                 let pixels = tw * th;
@@ -93,7 +93,7 @@ export default class HextileDecoder {
                 for (let i = 0;i <  pixels;i++) {
                     rQ[rQi + i * 4 + 3] = 255;
                 }
-                display.blitImage(tx, ty, tw, th, rQ, rQi);
+                display.blitImage(tx, ty, tw, th, rQ, rQi, frame_id);
                 rQi += bytes - 1;
             } else {
                 if (subencoding & 0x02) {  // Background
@@ -131,7 +131,7 @@ export default class HextileDecoder {
                         this._subTile(sx, sy, sw, sh, color);
                     }
                 }
-                this._finishTile(display);
+                this._finishTile(display, frame_id);
             }
             sock.rQi = rQi;
             this._lastsubencoding = subencoding;
@@ -183,9 +183,9 @@ export default class HextileDecoder {
     }
 
     // draw the current tile to the screen
-    _finishTile(display) {
+    _finishTile(display, frame_id) {
         display.blitImage(this._tileX, this._tileY,
                           this._tileW, this._tileH,
-                          this._tileBuffer, 0);
+                          this._tileBuffer, 0, frame_id);
     }
 }

core/decoders/qoi/decoder.js

@@ -0,0 +1,342 @@
+let wasm;
+const heap = new Array(32).fill(undefined);
+heap.push(undefined, null, true, false);
+
+function getObject(idx) {
+  return heap[idx];
+}
+let heap_next = heap.length;
+
+function dropObject(idx) {
+  if (idx < 36) return;
+  heap[idx] = heap_next;
+  heap_next = idx;
+}
+
+function takeObject(idx) {
+  const ret = getObject(idx);
+  dropObject(idx);
+  return ret;
+}
+const cachedTextDecoder = new TextDecoder('utf-8', {
+  ignoreBOM: true,
+  fatal: true
+});
+cachedTextDecoder.decode();
+let cachegetUint8Memory0 = null;
+
+function getUint8Memory0() {
+  if (cachegetUint8Memory0 === null || cachegetUint8Memory0.buffer !== wasm.memory.buffer) {
+    cachegetUint8Memory0 = new Uint8Array(wasm.memory.buffer);
+  }
+  return cachegetUint8Memory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+  return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
+let cachegetInt32Memory0 = null;
+
+function getInt32Memory0() {
+  if (cachegetInt32Memory0 === null || cachegetInt32Memory0.buffer !== wasm.memory.buffer) {
+    cachegetInt32Memory0 = new Int32Array(wasm.memory.buffer);
+  }
+  return cachegetInt32Memory0;
+}
+
+function getArrayU8FromWasm0(ptr, len) {
+  return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len);
+}
+
+let WASM_VECTOR_LEN = 0;
+
+function passArray8ToWasm0(arg, malloc) {
+  const ptr = malloc(arg.length * 1);
+  getUint8Memory0().set(arg, ptr / 1);
+  WASM_VECTOR_LEN = arg.length;
+  return ptr;
+}
+/**
+ * @param {Uint8Array} bytes
+ * @returns {ImageData}
+ */
+function decode_qoi(bytes) {
+  try {
+    const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+    const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc);
+    const len0 = WASM_VECTOR_LEN;
+    wasm.decode_qoi(retptr, ptr0, len0);
+    var r0 = getInt32Memory0()[retptr / 4 + 0];
+    var r1 = getInt32Memory0()[retptr / 4 + 1];
+    var r2 = getInt32Memory0()[retptr / 4 + 2];
+    if (r2) {
+      throw takeObject(r1);
+    }
+    return takeObject(r0);
+  } finally {
+    wasm.__wbindgen_add_to_stack_pointer(16);
+  }
+}
+
+function addHeapObject(obj) {
+  if (heap_next === heap.length) heap.push(heap.length + 1);
+  const idx = heap_next;
+  heap_next = heap[idx];
+
+  heap[idx] = obj;
+  return idx;
+}
+
+const cachedTextEncoder = new TextEncoder('utf-8');
+
+const encodeString = (typeof cachedTextEncoder.encodeInto === 'function' ?
+  function(arg, view) {
+    return cachedTextEncoder.encodeInto(arg, view);
+  } :
+  function(arg, view) {
+    const buf = cachedTextEncoder.encode(arg);
+    view.set(buf);
+    return {
+      read: arg.length,
+      written: buf.length
+    };
+  });
+
+function passStringToWasm0(arg, malloc, realloc) {
+
+  if (realloc === undefined) {
+    const buf = cachedTextEncoder.encode(arg);
+    const ptr = malloc(buf.length);
+    getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
+    WASM_VECTOR_LEN = buf.length;
+    return ptr;
+  }
+
+  let len = arg.length;
+  let ptr = malloc(len);
+
+  const mem = getUint8Memory0();
+
+  let offset = 0;
+
+  for (; offset < len; offset++) {
+    const code = arg.charCodeAt(offset);
+    if (code > 0x7F) break;
+    mem[ptr + offset] = code;
+  }
+
+  if (offset !== len) {
+    if (offset !== 0) {
+      arg = arg.slice(offset);
+    }
+    ptr = realloc(ptr, len, len = offset + arg.length * 3);
+    const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
+    const ret = encodeString(arg, view);
+
+    offset += ret.written;
+  }
+
+  WASM_VECTOR_LEN = offset;
+  return ptr;
+}
+
+let cachegetUint8ClampedMemory0 = null;
+
+function getUint8ClampedMemory0() {
+  if (cachegetUint8ClampedMemory0 === null || cachegetUint8ClampedMemory0.buffer !== wasm.memory.buffer) {
+    cachegetUint8ClampedMemory0 = new Uint8ClampedArray(wasm.memory.buffer);
+  }
+  return cachegetUint8ClampedMemory0;
+}
+
+function getClampedArrayU8FromWasm0(ptr, len) {
+  return getUint8ClampedMemory0().subarray(ptr / 1, ptr / 1 + len);
+}
+
+function handleError(f, args) {
+  try {
+    return f.apply(this, args);
+  } catch (e) {
+    wasm.__wbindgen_exn_store(addHeapObject(e));
+  }
+}
+/**
+ */
+class QoiImage {
+
+  static __wrap(ptr) {
+    const obj = Object.create(QoiImage.prototype);
+    obj.ptr = ptr;
+
+    return obj;
+  }
+
+  __destroy_into_raw() {
+    const ptr = this.ptr;
+    this.ptr = 0;
+
+    return ptr;
+  }
+
+  free() {
+    const ptr = this.__destroy_into_raw();
+    wasm.__wbg_qoiimage_free(ptr);
+  }
+  /**
+   */
+  constructor() {
+    const ret = wasm.qoiimage_new();
+    return QoiImage.__wrap(ret);
+  }
+  /**
+   * @returns {number}
+   */
+  get_width() {
+    const ret = wasm.qoiimage_get_width(this.ptr);
+    return ret >>> 0;
+  }
+  /**
+   * @returns {number}
+   */
+  get_height() {
+    const ret = wasm.qoiimage_get_height(this.ptr);
+    return ret >>> 0;
+  }
+  /**
+   * @returns {number}
+   */
+  get_channels() {
+    const ret = wasm.qoiimage_get_channels(this.ptr);
+    return ret;
+  }
+  /**
+   * @returns {Uint8Array}
+   */
+  get_bytes() {
+    try {
+      const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+      wasm.qoiimage_get_bytes(retptr, this.ptr);
+      var r0 = getInt32Memory0()[retptr / 4 + 0];
+      var r1 = getInt32Memory0()[retptr / 4 + 1];
+      var v0 = getArrayU8FromWasm0(r0, r1).slice();
+      wasm.__wbindgen_free(r0, r1 * 1);
+      return v0;
+    } finally {
+      wasm.__wbindgen_add_to_stack_pointer(16);
+    }
+  }
+}
+
+async function load(module, imports) {
+  if (typeof Response === 'function' && module instanceof Response) {
+    const bytes = await module.arrayBuffer();
+    return await WebAssembly.instantiate(bytes, imports);
+  } else {
+    const instance = await WebAssembly.instantiate(module, imports);
+
+    if (instance instanceof WebAssembly.Instance) {
+      return {
+        instance,
+        module
+      };
+
+    } else {
+      return instance;
+    }
+  }
+}
+
+async function init(input) {
+  if (typeof input === 'undefined') {
+    input = '/core/decoders/qoi/qoi_viewer_bg.wasm';
+  }
+  const imports = {};
+  imports.wbg = {};
+  imports.wbg.__wbg_new_693216e109162396 = function() {
+    const ret = new Error();
+    return addHeapObject(ret);
+  };
+  imports.wbg.__wbg_stack_0ddaca5d1abfb52f = function(arg0, arg1) {
+    const ret = getObject(arg1).stack;
+    const ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    getInt32Memory0()[arg0 / 4 + 1] = len0;
+    getInt32Memory0()[arg0 / 4 + 0] = ptr0;
+  };
+  imports.wbg.__wbg_error_09919627ac0992f5 = function(arg0, arg1) {
+    try {
+      console.error(getStringFromWasm0(arg0, arg1));
+    } finally {
+      wasm.__wbindgen_free(arg0, arg1);
+    }
+  };
+  imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
+    takeObject(arg0);
+  };
+  imports.wbg.__wbg_newwithu8clampedarrayandsh_87d2f0a48030f922 = function() {
+    return handleError(function(arg0, arg1, arg2, arg3) {
+      const ret = new ImageData(getClampedArrayU8FromWasm0(arg0, arg1), arg2 >>> 0, arg3 >>> 0);
+      return addHeapObject(ret);
+    }, arguments)
+  };
+  imports.wbg.__wbindgen_throw = function(arg0, arg1) {
+    throw new Error(getStringFromWasm0(arg0, arg1));
+  };
+  if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
+    input = fetch(input);
+  }
+  const {
+    instance,
+    module
+  } = await load(await input, imports);
+  wasm = instance.exports;
+  init.__wbindgen_wasm_module = module;
+  return wasm;
+}
+
+var arr;
+
+async function run() {
+  self.addEventListener('message', function(evt) {
+    try {
+      let length = evt.data.length;
+      let data = new Uint8Array(evt.data.sab.slice(0, length));
+      let resultData = decode_qoi(data);
+      if (!arr) {
+        arr = new Uint8Array(evt.data.sabR);
+      }
+      let lengthR = resultData.data.length;
+      arr.set(resultData.data);
+      let img = {
+        colorSpace: resultData.colorSpace,
+        width: resultData.width,
+        height: resultData.height
+      };
+      self.postMessage({
+        result: 0,
+        img: img,
+        length: lengthR,
+        width: evt.data.width,
+        height: evt.data.height,
+        x: evt.data.x,
+        y: evt.data.y,
+        frame_id: evt.data.frame_id
+      });
+    } catch (err) {
+      self.postMessage({
+        result: 2,
+        error: err
+      });
+    }
+  }, false);
+
+  await init();
+
+  //Send message that worker is ready
+  self.postMessage({
+    result: 1
+  })
+}
+
+run();

core/decoders/raw.js

@@ -12,7 +12,7 @@ export default class RawDecoder {
         this._lines = 0;
     }
 
-    decodeRect(x, y, width, height, sock, display, depth) {
+    decodeRect(x, y, width, height, sock, display, depth, frame_id) {
         if ((width === 0) || (height === 0)) {
             return true;
         }
@@ -54,7 +54,7 @@ export default class RawDecoder {
             data[i * 4 + 3] = 255;
         }
 
-        display.blitImage(x, curY, width, currHeight, data, index);
+        display.blitImage(x, curY, width, currHeight, data, index, frame_id);
         sock.rQskipBytes(currHeight * bytesPerLine);
         this._lines -= currHeight;
         if (this._lines > 0) {

core/decoders/rre.js

@@ -12,7 +12,7 @@ export default class RREDecoder {
         this._subrects = 0;
     }
 
-    decodeRect(x, y, width, height, sock, display, depth) {
+    decodeRect(x, y, width, height, sock, display, depth, frame_id) {
         if (this._subrects === 0) {
             if (sock.rQwait("RRE", 4 + 4)) {
                 return false;
@@ -34,7 +34,7 @@ export default class RREDecoder {
             let sy = sock.rQshift16();
             let swidth = sock.rQshift16();
             let sheight = sock.rQshift16();
-            display.fillRect(x + sx, y + sy, swidth, sheight, color);
+            display.fillRect(x + sx, y + sy, swidth, sheight, color, frame_id);
 
             this._subrects--;
         }

core/decoders/tight.js

@@ -12,12 +12,14 @@ import * as Log from '../util/logging.js';
 import Inflator from "../inflator.js";
 
 export default class TightDecoder {
-    constructor() {
+    constructor(display) {
         this._ctl = null;
         this._filter = null;
         this._numColors = 0;
         this._palette = new Uint8Array(1024);  // 256 * 4 (max palette size * max bytes-per-pixel)
         this._len = 0;
+        this._enableQOI = false;
+        this._displayGlobal = display;
 
         this._zlibs = [];
         for (let i = 0; i < 4; i++) {
@@ -25,7 +27,14 @@ export default class TightDecoder {
         }
     }
 
-    decodeRect(x, y, width, height, sock, display, depth) {
+    enableQOI() {
+        if (!this._enableQOI) {
+            this._enableQOIWorkers();
+        }
+        return this._enableQOI; //did it succeed
+    }
+
+    decodeRect(x, y, width, height, sock, display, depth, frame_id) {
         if (this._ctl === null) {
             if (sock.rQwait("TIGHT compression-control", 1)) {
                 return false;
@@ -37,7 +46,7 @@ export default class TightDecoder {
             for (let i = 0; i < 4; i++) {
                 if ((this._ctl >> i) & 1) {
                     this._zlibs[i].reset();
-                    Log.Debug("Reset zlib stream " + i);
+                    Log.Info("Reset zlib stream " + i);
                 }
             }
 
@@ -49,19 +58,22 @@ export default class TightDecoder {
 
         if (this._ctl === 0x08) {
             ret = this._fillRect(x, y, width, height,
-                                 sock, display, depth);
+                                 sock, display, depth, frame_id);
         } else if (this._ctl === 0x09) {
             ret = this._jpegRect(x, y, width, height,
-                                 sock, display, depth);
+                                 sock, display, depth, frame_id);
         } else if (this._ctl === 0x0A) {
             ret = this._pngRect(x, y, width, height,
-                                sock, display, depth);
+                                sock, display, depth, frame_id);
         } else if ((this._ctl & 0x08) == 0) {
             ret = this._basicRect(this._ctl, x, y, width, height,
-                                  sock, display, depth);
+                                  sock, display, depth, frame_id);
         } else if (this._ctl === 0x0B) {
             ret = this._webpRect(x, y, width, height,
-                                sock, display, depth);
+                                sock, display, depth, frame_id);
+        } else if (this._ctl === 0x0C) {
+            ret = this._qoiRect(x, y, width, height,
+                                sock, display, depth, frame_id);
         } else {
             throw new Error("Illegal tight compression received (ctl: " +
                                    this._ctl + ")");
@@ -74,7 +86,7 @@ export default class TightDecoder {
         return ret;
     }
 
-    _fillRect(x, y, width, height, sock, display, depth) {
+    _fillRect(x, y, width, height, sock, display, depth, frame_id) {
         if (sock.rQwait("TIGHT", 3)) {
             return false;
         }
@@ -83,39 +95,81 @@ export default class TightDecoder {
         const rQ = sock.rQ;
 
         display.fillRect(x, y, width, height,
-                         [rQ[rQi], rQ[rQi + 1], rQ[rQi + 2]], false);
+                         [rQ[rQi], rQ[rQi + 1], rQ[rQi + 2]], frame_id, false);
         sock.rQskipBytes(3);
 
         return true;
     }
 
-    _jpegRect(x, y, width, height, sock, display, depth) {
+    _jpegRect(x, y, width, height, sock, display, depth, frame_id) {
         let data = this._readData(sock);
         if (data === null) {
             return false;
         }
 
-        display.imageRect(x, y, width, height, "image/jpeg", data);
+        display.imageRect(x, y, width, height, "image/jpeg", data, frame_id);
 
         return true;
     }
 
-    _webpRect(x, y, width, height, sock, display, depth) {
+    _webpRect(x, y, width, height, sock, display, depth, frame_id) {
         let data = this._readData(sock);
         if (data === null) {
             return false;
         }
 
-        display.imageRect(x, y, width, height, "image/webp", data);
+        display.imageRect(x, y, width, height, "image/webp", data, frame_id);
 
         return true;
     }
 
-    _pngRect(x, y, width, height, sock, display, depth) {
+    _processRectQ() {
+        while (this._availableWorkers.length > 0 && this._qoiRects.length > 0) {
+            let i = this._availableWorkers.pop();
+            let worker = this._workers[i];
+            let rect = this._qoiRects.shift();
+            this._arrs[i].set(rect.data);
+            worker.postMessage({
+                length: rect.data.length,
+                x: rect.x,
+                y: rect.y,
+                width: rect.width,
+                height: rect.height,
+                depth: rect.depth,
+                sab: this._sabs[i],
+                sabR: this._sabsR[i],
+                frame_id: rect.frame_id
+            });
+        }
+    }
+
+    _qoiRect(x, y, width, height, sock, display, depth, frame_id) {
+        let data = this._readData(sock);
+        if (data === null) {
+            return false;
+        }
+
+        if (this._enableQOI) {
+            let dataClone = new Uint8Array(data);
+            let item = {x: x,y: y,width: width,height: height,data: dataClone,depth: depth, frame_id: frame_id};
+            if (this._qoiRects.length < 1000) {
+                this._qoiRects.push(item);
+                this._processRectQ();
+            } else {
+                Log.Warn("QOI queue exceeded limit.");
+                this._qoiRects.splice(0, 500);
+            }
+            
+        }
+
+        return true;
+    }
+
+    _pngRect(x, y, width, height, sock, display, depth, frame_id) {
         throw new Error("PNG received in standard Tight rect");
     }
 
-    _basicRect(ctl, x, y, width, height, sock, display, depth) {
+    _basicRect(ctl, x, y, width, height, sock, display, depth, frame_id) {
         if (this._filter === null) {
             if (ctl & 0x4) {
                 if (sock.rQwait("TIGHT", 1)) {
@@ -136,15 +190,15 @@ export default class TightDecoder {
         switch (this._filter) {
             case 0: // CopyFilter
                 ret = this._copyFilter(streamId, x, y, width, height,
-                                       sock, display, depth);
+                                       sock, display, depth, frame_id);
                 break;
             case 1: // PaletteFilter
                 ret = this._paletteFilter(streamId, x, y, width, height,
-                                          sock, display, depth);
+                                          sock, display, depth, frame_id);
                 break;
             case 2: // GradientFilter
                 ret = this._gradientFilter(streamId, x, y, width, height,
-                                           sock, display, depth);
+                                           sock, display, depth, frame_id);
                 break;
             default:
                 throw new Error("Illegal tight filter received (ctl: " +
@@ -158,7 +212,7 @@ export default class TightDecoder {
         return ret;
     }
 
-    _copyFilter(streamId, x, y, width, height, sock, display, depth) {
+    _copyFilter(streamId, x, y, width, height, sock, display, depth, frame_id) {
         const uncompressedSize = width * height * 3;
         let data;
 
@@ -191,12 +245,12 @@ export default class TightDecoder {
             rgbx[i + 3] = 255;  // Alpha
         }
 
-        display.blitImage(x, y, width, height, rgbx, 0, false);
+        display.blitImage(x, y, width, height, rgbx, 0, frame_id, false);
 
         return true;
     }
 
-    _paletteFilter(streamId, x, y, width, height, sock, display, depth) {
+    _paletteFilter(streamId, x, y, width, height, sock, display, depth, frame_id) {
         if (this._numColors === 0) {
             if (sock.rQwait("TIGHT palette", 1)) {
                 return false;
@@ -244,9 +298,9 @@ export default class TightDecoder {
 
         // Convert indexed (palette based) image data to RGB
         if (this._numColors == 2) {
-            this._monoRect(x, y, width, height, data, this._palette, display);
+            this._monoRect(x, y, width, height, data, this._palette, display, frame_id);
         } else {
-            this._paletteRect(x, y, width, height, data, this._palette, display);
+            this._paletteRect(x, y, width, height, data, this._palette, display, frame_id);
         }
 
         this._numColors = 0;
@@ -254,7 +308,7 @@ export default class TightDecoder {
         return true;
     }
 
-    _monoRect(x, y, width, height, data, palette, display) {
+    _monoRect(x, y, width, height, data, palette, display, frame_id) {
         // Convert indexed (palette based) image data to RGB
         // TODO: reduce number of calculations inside loop
         const dest = this._getScratchBuffer(width * height * 4);
@@ -284,10 +338,10 @@ export default class TightDecoder {
             }
         }
 
-        display.blitImage(x, y, width, height, dest, 0, false);
+        display.blitImage(x, y, width, height, dest, 0, frame_id, false);
     }
 
-    _paletteRect(x, y, width, height, data, palette, display) {
+    _paletteRect(x, y, width, height, data, palette, display, frame_id) {
         // Convert indexed (palette based) image data to RGB
         const dest = this._getScratchBuffer(width * height * 4);
         const total = width * height * 4;
@@ -299,10 +353,10 @@ export default class TightDecoder {
             dest[i + 3] = 255;
         }
 
-        display.blitImage(x, y, width, height, dest, 0, false);
+        display.blitImage(x, y, width, height, dest, 0, frame_id, false);
     }
 
-    _gradientFilter(streamId, x, y, width, height, sock, display, depth) {
+    _gradientFilter(streamId, x, y, width, height, sock, display, depth, frame_id) {
         throw new Error("Gradient filter not implemented");
     }
 
@@ -342,4 +396,63 @@ export default class TightDecoder {
         }
         return this._scratchBuffer;
     }
+
+    _enableQOIWorkers() {
+        let sabTest = typeof SharedArrayBuffer;
+        if (sabTest !== 'undefined') {
+            this._enableQOI = true;
+            if ((window.navigator.hardwareConcurrency) && (window.navigator.hardwareConcurrency > 8)) {
+                this._threads = window.navigator.hardwareConcurrency;
+            } else {
+                this._threads = 8;
+            }
+            this._workerEnabled = false;
+            this._workers = [];
+            this._availableWorkers = [];
+            this._sabs = [];
+            this._sabsR = [];
+            this._arrs = [];
+            this._arrsR = [];
+            this._qoiRects = [];
+            this._rectQlooping = false;
+            for (let i = 0; i < this._threads; i++) {
+                this._workers.push(new Worker("/core/decoders/qoi/decoder.js"));
+                this._sabs.push(new SharedArrayBuffer(300000));
+                this._sabsR.push(new SharedArrayBuffer(400000));
+                this._arrs.push(new Uint8Array(this._sabs[i]));
+                this._arrsR.push(new Uint8ClampedArray(this._sabsR[i]));
+                this._workers[i].onmessage = (evt) => {
+                    this._availableWorkers.push(i);
+                    switch(evt.data.result) {
+                        case 0:
+                            let data = new Uint8ClampedArray(evt.data.length);
+                            data.set(this._arrsR[i].slice(0, evt.data.length));
+                            let img = new ImageData(data, evt.data.img.width, evt.data.img.height, {colorSpace: evt.data.img.colorSpace});
+                            
+                            this._displayGlobal.blitQoi(
+                                evt.data.x,
+                                evt.data.y,
+                                evt.data.width,
+                                evt.data.height,
+                                img,
+                                0,
+                                evt.data.frame_id,
+                                false
+                            );
+                            this._processRectQ();
+                            break;
+                        case 1:
+                            Log.Info("QOI Worker is now available.");
+                            break;
+                        case 2:
+                            Log.Info("Error on worker: " + evt.error);
+                            break;
+                    }
+                };
+            }
+        } else {
+            this._enableQOI = false;
+            Log.Warn("Enabling QOI Failed, client not compatible.");
+        }
+    }
 }

core/decoders/tightpng.js

@@ -10,13 +10,13 @@
 import TightDecoder from './tight.js';
 
 export default class TightPNGDecoder extends TightDecoder {
-    _pngRect(x, y, width, height, sock, display, depth) {
+    _pngRect(x, y, width, height, sock, display, depth, frame_id) {
         let data = this._readData(sock);
         if (data === null) {
             return false;
         }
 
-        display.imageRect(x, y, width, height, "image/png", data);
+        display.imageRect(x, y, width, height, "image/png", data, frame_id);
 
         return true;
     }

core/decoders/udp.js

@@ -22,7 +22,7 @@ export default class UDPDecoder {
         }
     }
 
-    decodeRect(x, y, width, height, data, display, depth) {
+    decodeRect(x, y, width, height, data, display, depth, frame_id) {
         let ctl = data[12];
         ctl = ctl >> 4;
 

core/display.js

@@ -9,12 +9,31 @@
 import * as Log from './util/logging.js';
 import Base64 from "./base64.js";
 import { toSigned32bit } from './util/int.js';
+import { isWindows } from './util/browser.js';
 
 export default class Display {
     constructor(target) {
-        this._renderQ = [];  // queue drawing actions for in-oder rendering
+        Log.Debug(">> Display.constructor");
-        this._currentFrame = [];
+
-        this._nextFrame = [];
+        /*
+        For performance reasons we use a multi dimensional array
+        1st Dimension of Array Represents Frames, each element is a Frame
+        2nd Dimension contains 4 elements
+            0 - int, FrameID
+            1 - int, Rect Count
+            2 - Array of Rect objects
+            3 - bool, is the frame complete
+            4 - int, index of current rect (post-processing)
+        */
+        this._asyncFrameQueue = [];
+        /*
+        Buffer for incoming frames. The larger the buffer the more time there is to collect, process, and order rects
+        but the more delay there is. May need to adjust this higher for lower power devices when UDP is complete.
+        Decoders that use WASM in parallel can also cause out of order rects
+        */
+        this._maxAsyncFrameQueue = 3;
+        this._clearAsyncQueue();
+
         this._flushing = false;
 
         // the full frame buffer (logical canvas) size
@@ -22,12 +41,7 @@ export default class Display {
         this._fbHeight = 0;
 
         this._renderMs = 0;
-
         this._prevDrawStyle = "";
-
-        Log.Debug(">> Display.constructor");
-
-        // The visible canvas
         this._target = target;
 
         if (!this._target) {
@@ -49,20 +63,22 @@ export default class Display {
 
         Log.Debug("User Agent: " + navigator.userAgent);
 
-        // performance metrics, try to calc a fps equivelant
+        // performance metrics
         this._flipCnt = 0;
         this._lastFlip = Date.now();
+        this._droppedFrames = 0;
+        this._droppedRects = 0;
+        this._missingRectCnt = 0;
         setInterval(function() {
             let delta = Date.now() - this._lastFlip;
             if (delta > 0) {
                 this._fps = (this._flipCnt / (delta / 1000)).toFixed(2);
             }
+            Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Missing Rect Cnt: ' + this._missingRectCnt);
             this._flipCnt = 0;
             this._lastFlip = Date.now();
         }.bind(this), 5000);
 
-        Log.Debug("<< Display.constructor");
-
         // ===== PROPERTIES =====
 
         this._scale = 1.0;
@@ -73,6 +89,11 @@ export default class Display {
         // ===== EVENT HANDLERS =====
 
         this.onflush = () => {  }; // A flush request has finished
+
+        // Use requestAnimationFrame to write to canvas, to match display refresh rate
+        window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
+
+        Log.Debug("<< Display.constructor");
     }
 
     // ===== PROPERTIES =====
@@ -149,8 +170,6 @@ export default class Display {
             return;
         }
         Log.Debug("viewportChange deltaX: " + deltaX + ", deltaY: " + deltaY);
-
-        this.flip();
     }
 
     viewportChangeSize(width, height) {
@@ -186,8 +205,6 @@ export default class Display {
             // The position might need to be updated if we've grown
             this.viewportChangePos(0, 0);
 
-            this.flip();
-
             // Update the visible size of the target canvas
             this._rescale(this._scale);
         }
@@ -243,54 +260,37 @@ export default class Display {
     }
 
     // rendering canvas
-    flip(fromQueue) {
+    flip(frame_id, rect_cnt) {
-        if (!fromQueue) {
+        this._asyncRenderQPush({
-            this._renderQPush({
+            'type': 'flip',
-                'type': 'flip'
+            'frame_id': frame_id,
-            });
+            'rect_cnt': rect_cnt
-        } else {
+        });
-            for (let i = 0; i < this._currentFrame.length; i++) {
-                const a = this._currentFrame[i];
-                switch (a.type) {
-                    case 'copy':
-                        this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, true);
-                        break;
-                    case 'fill':
-                        this.fillRect(a.x, a.y, a.width, a.height, a.color, true);
-                        break;
-                    case 'blit':
-                        this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, true);
-                        break;
-                    case 'img':
-                        this.drawImage(a.img, a.x, a.y, a.width, a.height);
-                        break;
-                }
-            }
-            this._flipCnt += 1;
-        }
     }
 
     pending() {
-        return this._renderQ.length > 0;
+        //is the slot in the queue for the newest frame in use
+        return this._asyncFrameQueue[this._maxAsyncFrameQueue - 1][0] > 0;
     }
 
     flush() {
-        if (this._renderQ.length === 0) {
+        //force oldest frame to render
-            this.onflush();
+        this._asyncFrameComplete(0, true);
-        } else {
+
-            this._flushing = true;
+        //this in effect blocks more incoming frames until the oldest frame has been rendered to canvas (tcp only)
-        }
+        this._flushing = true;
     }
 
-    fillRect(x, y, width, height, color, fromQueue) {
+    fillRect(x, y, width, height, color, frame_id, fromQueue) {
         if (!fromQueue) {
-            this._renderQPush({
+            this._asyncRenderQPush({
                 'type': 'fill',
                 'x': x,
                 'y': y,
                 'width': width,
                 'height': height,
-                'color': color
+                'color': color,
+                'frame_id': frame_id
             });
         } else {
             this._setFillColor(color);
@@ -298,9 +298,9 @@ export default class Display {
         }
     }
 
-    copyImage(oldX, oldY, newX, newY, w, h, fromQueue) {
+    copyImage(oldX, oldY, newX, newY, w, h, frame_id, fromQueue) {
         if (!fromQueue) {
-            this._renderQPush({
+            this._asyncRenderQPush({
                 'type': 'copy',
                 'oldX': oldX,
                 'oldY': oldY,
@@ -308,6 +308,7 @@ export default class Display {
                 'y': newY,
                 'width': w,
                 'height': h,
+                'frame_id': frame_id
             });
         } else {
             // Due to this bug among others [1] we need to disable the image-smoothing to
@@ -328,39 +329,40 @@ export default class Display {
         }
     }
 
-    imageRect(x, y, width, height, mime, arr) {
+    imageRect(x, y, width, height, mime, arr, frame_id) {
         /* The internal logic cannot handle empty images, so bail early */
         if ((width === 0) || (height === 0)) {
             return;
         }
-
         const img = new Image();
         img.src = "data: " + mime + ";base64," + Base64.encode(arr);
 
-        this._renderQPush({
+        this._asyncRenderQPush({
             'type': 'img',
             'img': img,
             'x': x,
             'y': y,
             'width': width,
-            'height': height
+            'height': height,
+            'frame_id': frame_id
         });
     }
 
-    blitImage(x, y, width, height, arr, offset, fromQueue) {
+    blitImage(x, y, width, height, arr, offset, frame_id, fromQueue) {
         if (!fromQueue) {
             // NB(directxman12): it's technically more performant here to use preallocated arrays,
             // but it's a lot of extra work for not a lot of payoff -- if we're using the render queue,
             // this probably isn't getting called *nearly* as much
             const newArr = new Uint8Array(width * height * 4);
             newArr.set(new Uint8Array(arr.buffer, 0, newArr.length));
-            this._renderQPush({
+            this._asyncRenderQPush({
                 'type': 'blit',
                 'data': newArr,
                 'x': x,
                 'y': y,
                 'width': width,
                 'height': height,
+                'frame_id': frame_id
             });
         } else {
             // NB(directxman12): arr must be an Type Array view
@@ -372,9 +374,25 @@ export default class Display {
         }
     }
 
+    blitQoi(x, y, width, height, arr, offset, frame_id, fromQueue) {
+        if (!fromQueue) {
+            this._asyncRenderQPush({
+                'type': 'blitQ',
+                'data': arr,
+                'x': x,
+                'y': y,
+                'width': width,
+                'height': height,
+                'frame_id': frame_id
+            });
+        } else {
+            this._targetCtx.putImageData(arr, x, y);
+        }
+    }
+
     drawImage(img, x, y, w, h) {
         try {
-	    if (img.width != w || img.height != h) {
+            if (img.width != w || img.height != h) {
                 this._targetCtx.drawImage(img, x, y, w, h);
             } else {
                 this._targetCtx.drawImage(img, x, y);
@@ -407,6 +425,144 @@ export default class Display {
 
     // ===== PRIVATE METHODS =====
 
+    /*
+    Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding
+    */
+    _asyncRenderQPush(rect) {
+        let frameIx = -1;
+        let oldestFrameID = Number.MAX_SAFE_INTEGER;
+        let newestFrameID = 0;
+        for (let i=0; i<this._maxAsyncFrameQueue; i++) {
+            if (rect.frame_id == this._asyncFrameQueue[i][0]) {
+                this._asyncFrameQueue[i][2].push(rect);
+                frameIx = i;
+                break;
+            } else if (this._asyncFrameQueue[i][0] == 0) {
+                let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0);
+                this._asyncFrameQueue[i][0] = rect.frame_id;
+                this._asyncFrameQueue[i][1] = rect_cnt;
+                this._asyncFrameQueue[i][2].push(rect);
+                this._asyncFrameQueue[i][3] = (rect_cnt == 1);
+                frameIx = i;
+                break;
+            }
+            oldestFrameID = Math.min(oldestFrameID, this._asyncFrameQueue[i][0]);
+            newestFrameID = Math.max(newestFrameID, this._asyncFrameQueue[i][0]);
+        }
+
+        if (frameIx >= 0) {
+            if (rect.type == "flip") {
+                //flip rect contains the rect count for the frame
+                this._asyncFrameQueue[frameIx][1] = rect.rect_cnt;
+            }
+
+            if (this._asyncFrameQueue[frameIx][1] == this._asyncFrameQueue[frameIx][2].length) {
+                //frame is complete
+                this._asyncFrameComplete(frameIx);
+            }
+        } else {
+            if (rect.frame_id < oldestFrameID) {
+                //rect is older than any frame in the queue, drop it
+                this._droppedRects++;
+                return;
+            } else if (rect.frame_id > newestFrameID) {
+                //frame is newer than any frame in the queue, drop old frames
+                this._asyncFrameQueue.shift();
+                let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0);
+                this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0 ]);
+                this._droppedFrames++;
+            }
+        }
+        
+    }
+
+    /*
+    Clear the async frame buffer
+    */
+    _clearAsyncQueue() {
+        this._droppedFrames += this._asyncFrameQueue.length;
+
+        this._asyncFrameQueue = [];
+        for (let i=0; i<this._maxAsyncFrameQueue; i++) {
+            this._asyncFrameQueue.push([ 0, 0, [], false, 0 ])
+        }
+    }
+
+    /*
+    Pre-processing required before displaying a finished frame
+    If marked force, unloaded images will be skipped and the frame will be marked complete and ready for rendering
+    */
+    _asyncFrameComplete(frameIx, force=false) {
+        let currentFrameRectIx = this._asyncFrameQueue[frameIx][4];
+
+        if (force) {
+            if (this._asyncFrameQueue[frameIx][1] == 0) {
+                this._missingRectCnt++;
+            } else if (this._asyncFrameQueue[frameIx][1] !== this._asyncFrameQueue[frameIx][2].length) {
+                this._droppedRects += (this._asyncFrameQueue[frameIx][1] - this._asyncFrameQueue[frameIx][2].length);
+            }
+            while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) {   
+                if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
+                    this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip';
+                    this._droppedRects++;
+                }
+                currentFrameRectIx++;
+            }
+        } else {
+            while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) {
+                if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
+                    this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.addEventListener('load', () => { this._asyncFrameComplete(frameIx); });
+                    this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
+                    return;
+                }
+                currentFrameRectIx++;
+            }
+        }
+        this._asyncFrameQueue[frameIx][4] = currentFrameRectIx;
+        this._asyncFrameQueue[frameIx][3] = true;
+    }
+
+    /*
+    Push the oldest frame in the buffer to the canvas if it is marked ready
+    */
+    _pushAsyncFrame() {
+        if (this._asyncFrameQueue[0][3]) {
+            let frame = this._asyncFrameQueue.shift()[2];
+            this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]);
+            
+            //render the selected frame
+            for (let i = 0; i < frame.length; i++) {
+                
+                const a = frame[i];
+                switch (a.type) {
+                    case 'copy':
+                        this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, a.frame_id, true);
+                        break;
+                    case 'fill':
+                        this.fillRect(a.x, a.y, a.width, a.height, a.color, a.frame_id, true);
+                        break;
+                    case 'blit':
+                        this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true);
+                        break;
+                    case 'blitQ':
+                        this.blitQoi(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true);
+                        break;
+                    case 'img':
+                        this.drawImage(a.img, a.x, a.y, a.width, a.height);
+                        break;
+                }
+            }
+            this._flipCnt += 1;
+
+            if (this._flushing) {
+                this._flushing = false;
+                this.onflush();
+            }
+        }
+
+        window.requestAnimationFrame( () => { this._pushAsyncFrame(); });
+    }
+
     _rescale(factor) {
         this._scale = factor;
         const vp = this._viewportLoc;
@@ -445,56 +601,4 @@ export default class Display {
             this._prevDrawStyle = newStyle;
         }
     }
-
-    _renderQPush(action) {
-        this._renderQ.push(action);
-        if (this._renderQ.length === 1) {
-            this._scanRenderQ();
-        }
-    }
-
-    _resumeRenderQ() {
-        this.removeEventListener('load', this._noVNCDisplay._resumeRenderQ);
-        this._noVNCDisplay._scanRenderQ();
-    }
-
-    _scanRenderQ() {
-        let ready = true;
-        let before = Date.now();
-        while (ready && this._renderQ.length > 0) {
-            const a = this._renderQ[0];
-            switch (a.type) {
-                case 'flip':
-                    this._currentFrame = this._nextFrame;
-                    this._nextFrame = [];
-                    this.flip(true);
-                    break;
-                case 'img':
-                    if (a.img.complete) {
-                        this._nextFrame.push(a);
-                    } else {
-                        a.img._noVNCDisplay = this;
-                        a.img.addEventListener('load', this._resumeRenderQ);
-                        // We need to wait for this image to 'load'
-                        ready = false;
-                    }
-                    break;
-                default:
-                    this._nextFrame.push(a);
-                    break;
-            }
-
-            if (ready) {
-                this._renderQ.shift();
-            }
-        }
-
-        if (this._renderQ.length === 0 && this._flushing) {
-            this._flushing = false;
-            this.onflush();
-        }
-
-        let elapsed = Date.now() - before;
-        this._renderMs += elapsed;
-    }
 }

core/encodings.js

@@ -53,6 +53,7 @@ export const encodings = {
     pseudoEncodingVideoScalingLevel9: -1987,
     pseudoEncodingVideoOutTimeLevel1: -1986,
     pseudoEncodingVideoOutTimeLevel100: -1887,
+    pseudoEncodingQOI: -1886,
 
     pseudoEncodingVMwareCursor: 0x574d5664,
     pseudoEncodingVMwareCursorPosition: 0x574d5666,

core/rfb.js

@@ -43,6 +43,7 @@ const DEFAULT_BACKGROUND = 'rgb(40, 40, 40)';
 
 var _videoQuality =  2;
 var _enableWebP = false;
+var _enableQOI = false;
 
 // Minimum wait (ms) between two mouse moves
 const MOUSE_MOVE_DELAY = 17; 
@@ -139,6 +140,7 @@ export default class RFB extends EventTargetMixin {
         this._maxVideoResolutionY = 540;
         this._clipboardBinary = true;
         this._useUdp = true;
+        this._enableQOI = false;
         this.TransitConnectionStates = {
             Tcp: Symbol("tcp"),
             Udp: Symbol("udp"),
@@ -173,12 +175,14 @@ export default class RFB extends EventTargetMixin {
         this._decoders = {};
 
         this._FBU = {
-            rects: 0,
+            rects: 0, // current rect number
             x: 0,
             y: 0,
             width: 0,
             height: 0,
             encoding: null,
+            frame_id: 0,
+            rect_total: 0, //Total rects in frame
         };
 
         // Mouse state
@@ -248,15 +252,6 @@ export default class RFB extends EventTargetMixin {
         // initial cursor instead.
         this._cursorImage = RFB.cursors.none;
 
-        // populate decoder array with objects
-        this._decoders[encodings.encodingRaw] = new RawDecoder();
-        this._decoders[encodings.encodingCopyRect] = new CopyRectDecoder();
-        this._decoders[encodings.encodingRRE] = new RREDecoder();
-        this._decoders[encodings.encodingHextile] = new HextileDecoder();
-        this._decoders[encodings.encodingTight] = new TightDecoder();
-        this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder();
-        this._decoders[encodings.encodingUDP] = new UDPDecoder();
-
         // NB: nothing that needs explicit teardown should be done
         // before this point, since this can throw an exception
         try {
@@ -267,6 +262,15 @@ export default class RFB extends EventTargetMixin {
         }
         this._display.onflush = this._onFlush.bind(this);
 
+        // populate decoder array with objects
+        this._decoders[encodings.encodingRaw] = new RawDecoder();
+        this._decoders[encodings.encodingCopyRect] = new CopyRectDecoder();
+        this._decoders[encodings.encodingRRE] = new RREDecoder();
+        this._decoders[encodings.encodingHextile] = new HextileDecoder();
+        this._decoders[encodings.encodingTight] = new TightDecoder(this._display);
+        this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder();
+        this._decoders[encodings.encodingUDP] = new UDPDecoder();
+
         this._keyboard = new Keyboard(this._canvas, touchInput);
         this._keyboard.onkeyevent = this._handleKeyEvent.bind(this);
 
@@ -481,6 +485,21 @@ export default class RFB extends EventTargetMixin {
         this._pendingApplyEncodingChanges = true;
     }
 
+    get enableQOI() { return this._enableQOI; }
+    set enableQOI(enabled) {
+        if(this._enableQOI === enabled) {
+            return;
+        }
+        if (enabled) {
+            if (!this._decoders[encodings.encodingTight].enableQOI()) {
+                //enabling qoi failed
+                return;
+            }
+        }
+        this._enableQOI = enabled;
+        this._pendingApplyEncodingChanges = true;
+    }
+
     get antiAliasing() { return this._display.antiAliasing; }
     set antiAliasing(value) {
        this._display.antiAliasing = value;
@@ -2520,12 +2539,8 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.encodingRaw);
 
         // Psuedo-encoding settings
-        var quality = 6;
-        var compression = 2;
-        var screensize = this._screenSize(false);
         encs.push(encodings.pseudoEncodingQualityLevel0 + this._qualityLevel);
         encs.push(encodings.pseudoEncodingCompressLevel0 + this._compressionLevel);
-
         encs.push(encodings.pseudoEncodingDesktopSize);
         encs.push(encodings.pseudoEncodingLastRect);
         encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent);
@@ -2537,6 +2552,9 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.pseudoEncodingExtendedClipboard);
         if (this._hasWebp())
             encs.push(encodings.pseudoEncodingWEBP);
+        if (this._enableQOI)
+            encs.push(encodings.pseudoEncodingQOI);
+            
 
         // kasm settings; the server may be configured to ignore these
         encs.push(encodings.pseudoEncodingJpegVideoQualityLevel0 + this.jpegVideoQuality);
@@ -3053,11 +3071,11 @@ export default class RFB extends EventTargetMixin {
             encoding: parseInt((data[8] << 24) + (data[9] << 16) +
                                             (data[10] << 8) + data[11], 10)
         };
-
+        
         switch (frame.encoding) {
             case encodings.pseudoEncodingLastRect:
                 if (document.visibilityState !== "hidden") {
-                    this._display.flip();
+                    this._display.flip(false); //TODO: UDP is now broken, flip needs rect count and frame number
                     this._udpBuffer.clear();
                 }
                 break;
@@ -3158,6 +3176,9 @@ export default class RFB extends EventTargetMixin {
             this._sock.rQskipBytes(1);  // Padding
             this._FBU.rects = this._sock.rQshift16();
 
+            this._FBU.frame_id++;
+            this._FBU.rect_total = 0;
+
             // Make sure the previous frame is fully rendered first
             // to avoid building up an excessive queue
             if (this._display.pending()) {
@@ -3180,7 +3201,8 @@ export default class RFB extends EventTargetMixin {
                 this._FBU.encoding = parseInt((hdr[8] << 24) + (hdr[9] << 16) +
                                               (hdr[10] << 8) + hdr[11], 10);
             }
-
+            
+            
             if (!this._handleRect()) {
                 return false;
             }
@@ -3189,14 +3211,17 @@ export default class RFB extends EventTargetMixin {
             this._FBU.encoding = null;
         }
 
-        this._display.flip();
+        if (this._FBU.rect_total > 0) {
-
+            this._display.flip(this._FBU.frame_id, this._FBU.rect_total);
+        }
+        
         return true;  // We finished this FBU
     }
 
     _handleRect() {
         switch (this._FBU.encoding) {
             case encodings.pseudoEncodingLastRect:
+                this._FBU.rect_total++; //only track rendered rects and last rect
                 this._FBU.rects = 1; // Will be decreased when we return
                 return true;
 
@@ -3224,7 +3249,11 @@ export default class RFB extends EventTargetMixin {
                 return this._handleExtendedDesktopSize();
 
             default:
-                return this._handleDataRect();
+                if (this._handleDataRect()) {
+                    this._FBU.rect_total++; //only track rendered rects and last rect
+                    return true;
+                } 
+                return false;
         }
     }
 
@@ -3524,9 +3553,9 @@ export default class RFB extends EventTargetMixin {
             return decoder.decodeRect(this._FBU.x, this._FBU.y,
                                       this._FBU.width, this._FBU.height,
                                       this._sock, this._display,
-                                      this._fbDepth);
+                                      this._fbDepth, this._FBU.frame_id);
         } catch (err) {
-	    this._fail("Error decoding rect: " + err);
+            this._fail("Error decoding rect: " + err);
             return false;
         }
     }

vnc.html

@@ -277,6 +277,12 @@
                                         <span class="slider-label">IME Input Mode</span>
                                     </label>
                                 </li>
+                                <li>
+                                    <label class="switch"><input id="noVNC_setting_enable_qoi" type="checkbox" />
+                                        <span class="slider round"></span>
+                                        <span class="slider-label">QOI Lossless</span>
+                                    </label>
+                                </li>
                                 <li>
                                     <label class="switch"><input id="noVNC_setting_virtual_keyboard_visible" type="checkbox" />
                                         <span class="slider round"></span>