diff --git a/app/ui.js b/app/ui.js index 206018ab..d253ce24 100644 --- a/app/ui.js +++ b/app/ui.js @@ -245,6 +245,7 @@ const UI = { UI.initSetting('enable_perf_stats', false); UI.initSetting('virtual_keyboard_visible', false); UI.initSetting('enable_ime', false); + UI.initSetting('enable_qoi', false); UI.initSetting('enable_webrtc', false); UI.toggleKeyboardControls(); @@ -558,6 +559,8 @@ const UI = { UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls); UI.addSettingChangeHandler('enable_ime'); UI.addSettingChangeHandler('enable_ime', UI.toggleIMEMode); + UI.addSettingChangeHandler('enable_qoi'); + UI.addSettingChangeHandler('enable_qoi', UI.toggleQOI); UI.addSettingChangeHandler('enable_webrtc'); UI.addSettingChangeHandler('enable_webrtc', UI.toggleWebRTC); }, @@ -1399,6 +1402,7 @@ const UI = { UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless'); UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime'); UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless; + UI.rfb.enableQOI = UI.getSetting('enable_qoi'); UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc'); UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper(); @@ -1651,6 +1655,18 @@ const UI = { UI.toggleIMEMode(); } break; + case 'disable_qoi': + if(UI.getSetting('enable_qoi')) { + UI.forceSetting('enable_qoi', false, false); + } + UI.toggleQOI(); + break; + case 'enable_qoi': + if(!UI.getSetting('enable_qoi')) { + UI.forceSetting('enable_qoi', true, false); + } + UI.toggleQOI(); + break; case 'enable_webrtc': if (!UI.getSetting('enable_webrtc')) { UI.forceSetting('enable_webrtc', true, false); @@ -1961,6 +1977,8 @@ const UI = { UI.enableSetting('video_out_time'); UI.showStatus("Refresh or reconnect to apply changes."); return; + case 5: //extreme+lossless + UI.forceSetting('enable_qoi', true, false); case 4: //extreme UI.forceSetting('dynamic_quality_min', 9); UI.forceSetting('dynamic_quality_max', 9); @@ -2024,6 +2042,12 @@ const UI = { break; } + //force QOI off if mode is below extreme + if (present_mode !== 4 && UI.getSetting('enable_qoi')) { + UI.showStatus("Lossless QOI disabled when not in extreme quality mode."); + UI.forceSetting('enable_qoi', false, false); + } + if (UI.rfb) { UI.rfb.qualityLevel = parseInt(UI.getSetting('quality')); UI.rfb.antiAliasing = parseInt(UI.getSetting('anti_aliasing')); @@ -2041,6 +2065,7 @@ const UI = { UI.rfb.frameRate = parseInt(UI.getSetting('framerate')); UI.rfb.enableWebP = UI.getSetting('enable_webp'); UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality')); + UI.rfb.enableQOI = UI.getSetting('enable_qoi'); // Gracefully update settings server side UI.rfb.updateConnectionSettings(); @@ -2096,9 +2121,27 @@ const UI = { } else { UI.rfb.enableWebRTC = false; } + UI.updateQuality(); } }, + toggleQOI() { + if(UI.rfb) { + if(UI.getSetting('enable_qoi')) { + UI.rfb.enableQOI = true; + if (!UI.rfb.enableQOI) { + UI.showStatus("Enabling QOI failed, browser may not be compatible with WASM and/or Workers."); + UI.forceSetting('enable_qoi', false, false); + return; + } + UI.forceSetting('video_quality', 4, false); //force into extreme quality mode + } else { + UI.rfb.enableQOI = false; + } + UI.updateQuality(); + } + }, + showKeyboardControls() { document.getElementById('noVNC_keyboard_control').classList.add("is-visible"); }, diff --git a/core/decoders/copyrect.js b/core/decoders/copyrect.js index 9e6391a1..55fd1b34 100644 --- a/core/decoders/copyrect.js +++ b/core/decoders/copyrect.js @@ -8,7 +8,8 @@ */ export default class CopyRectDecoder { - decodeRect(x, y, width, height, sock, display, depth) { + + decodeRect(x, y, width, height, sock, display, depth, frame_id) { if (sock.rQwait("COPYRECT", 4)) { return false; } @@ -20,7 +21,7 @@ export default class CopyRectDecoder { return true; } - display.copyImage(deltaX, deltaY, x, y, width, height); + display.copyImage(deltaX, deltaY, x, y, width, height, frame_id); return true; } diff --git a/core/decoders/hextile.js b/core/decoders/hextile.js index ac21eff0..557e1036 100644 --- a/core/decoders/hextile.js +++ b/core/decoders/hextile.js @@ -16,7 +16,7 @@ export default class HextileDecoder { this._tileBuffer = new Uint8Array(16 * 16 * 4); } - decodeRect(x, y, width, height, sock, display, depth) { + decodeRect(x, y, width, height, sock, display, depth, frame_id) { if (this._tiles === 0) { this._tilesX = Math.ceil(width / 16); this._tilesY = Math.ceil(height / 16); @@ -85,7 +85,7 @@ export default class HextileDecoder { // Weird: ignore blanks are RAW Log.Debug(" Ignoring blank after RAW"); } else { - display.fillRect(tx, ty, tw, th, this._background); + display.fillRect(tx, ty, tw, th, this._background, frame_id); } } else if (subencoding & 0x01) { // Raw let pixels = tw * th; @@ -93,7 +93,7 @@ export default class HextileDecoder { for (let i = 0;i < pixels;i++) { rQ[rQi + i * 4 + 3] = 255; } - display.blitImage(tx, ty, tw, th, rQ, rQi); + display.blitImage(tx, ty, tw, th, rQ, rQi, frame_id); rQi += bytes - 1; } else { if (subencoding & 0x02) { // Background @@ -131,7 +131,7 @@ export default class HextileDecoder { this._subTile(sx, sy, sw, sh, color); } } - this._finishTile(display); + this._finishTile(display, frame_id); } sock.rQi = rQi; this._lastsubencoding = subencoding; @@ -183,9 +183,9 @@ export default class HextileDecoder { } // draw the current tile to the screen - _finishTile(display) { + _finishTile(display, frame_id) { display.blitImage(this._tileX, this._tileY, this._tileW, this._tileH, - this._tileBuffer, 0); + this._tileBuffer, 0, frame_id); } } diff --git a/core/decoders/qoi/decoder.js b/core/decoders/qoi/decoder.js new file mode 100644 index 00000000..eab72f13 --- /dev/null +++ b/core/decoders/qoi/decoder.js @@ -0,0 +1,342 @@ +let wasm; +const heap = new Array(32).fill(undefined); +heap.push(undefined, null, true, false); + +function getObject(idx) { + return heap[idx]; +} +let heap_next = heap.length; + +function dropObject(idx) { + if (idx < 36) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} +const cachedTextDecoder = new TextDecoder('utf-8', { + ignoreBOM: true, + fatal: true +}); +cachedTextDecoder.decode(); +let cachegetUint8Memory0 = null; + +function getUint8Memory0() { + if (cachegetUint8Memory0 === null || cachegetUint8Memory0.buffer !== wasm.memory.buffer) { + cachegetUint8Memory0 = new Uint8Array(wasm.memory.buffer); + } + return cachegetUint8Memory0; +} + +function getStringFromWasm0(ptr, len) { + return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)); +} + +let cachegetInt32Memory0 = null; + +function getInt32Memory0() { + if (cachegetInt32Memory0 === null || cachegetInt32Memory0.buffer !== wasm.memory.buffer) { + cachegetInt32Memory0 = new Int32Array(wasm.memory.buffer); + } + return cachegetInt32Memory0; +} + +function getArrayU8FromWasm0(ptr, len) { + return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len); +} + +let WASM_VECTOR_LEN = 0; + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1); + getUint8Memory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} +/** + * @param {Uint8Array} bytes + * @returns {ImageData} + */ +function decode_qoi(bytes) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_malloc); + const len0 = WASM_VECTOR_LEN; + wasm.decode_qoi(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var r2 = getInt32Memory0()[retptr / 4 + 2]; + if (r2) { + throw takeObject(r1); + } + return takeObject(r0); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +const cachedTextEncoder = new TextEncoder('utf-8'); + +const encodeString = (typeof cachedTextEncoder.encodeInto === 'function' ? + function(arg, view) { + return cachedTextEncoder.encodeInto(arg, view); + } : + function(arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }); + +function passStringToWasm0(arg, malloc, realloc) { + + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length); + getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len); + + const mem = getUint8Memory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3); + const view = getUint8Memory0().subarray(ptr + offset, ptr + len); + const ret = encodeString(arg, view); + + offset += ret.written; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +let cachegetUint8ClampedMemory0 = null; + +function getUint8ClampedMemory0() { + if (cachegetUint8ClampedMemory0 === null || cachegetUint8ClampedMemory0.buffer !== wasm.memory.buffer) { + cachegetUint8ClampedMemory0 = new Uint8ClampedArray(wasm.memory.buffer); + } + return cachegetUint8ClampedMemory0; +} + +function getClampedArrayU8FromWasm0(ptr, len) { + return getUint8ClampedMemory0().subarray(ptr / 1, ptr / 1 + len); +} + +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + wasm.__wbindgen_exn_store(addHeapObject(e)); + } +} +/** + */ +class QoiImage { + + static __wrap(ptr) { + const obj = Object.create(QoiImage.prototype); + obj.ptr = ptr; + + return obj; + } + + __destroy_into_raw() { + const ptr = this.ptr; + this.ptr = 0; + + return ptr; + } + + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_qoiimage_free(ptr); + } + /** + */ + constructor() { + const ret = wasm.qoiimage_new(); + return QoiImage.__wrap(ret); + } + /** + * @returns {number} + */ + get_width() { + const ret = wasm.qoiimage_get_width(this.ptr); + return ret >>> 0; + } + /** + * @returns {number} + */ + get_height() { + const ret = wasm.qoiimage_get_height(this.ptr); + return ret >>> 0; + } + /** + * @returns {number} + */ + get_channels() { + const ret = wasm.qoiimage_get_channels(this.ptr); + return ret; + } + /** + * @returns {Uint8Array} + */ + get_bytes() { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.qoiimage_get_bytes(retptr, this.ptr); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var v0 = getArrayU8FromWasm0(r0, r1).slice(); + wasm.__wbindgen_free(r0, r1 * 1); + return v0; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } +} + +async function load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { + instance, + module + }; + + } else { + return instance; + } + } +} + +async function init(input) { + if (typeof input === 'undefined') { + input = '/core/decoders/qoi/qoi_viewer_bg.wasm'; + } + const imports = {}; + imports.wbg = {}; + imports.wbg.__wbg_new_693216e109162396 = function() { + const ret = new Error(); + return addHeapObject(ret); + }; + imports.wbg.__wbg_stack_0ddaca5d1abfb52f = function(arg0, arg1) { + const ret = getObject(arg1).stack; + const ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + getInt32Memory0()[arg0 / 4 + 1] = len0; + getInt32Memory0()[arg0 / 4 + 0] = ptr0; + }; + imports.wbg.__wbg_error_09919627ac0992f5 = function(arg0, arg1) { + try { + console.error(getStringFromWasm0(arg0, arg1)); + } finally { + wasm.__wbindgen_free(arg0, arg1); + } + }; + imports.wbg.__wbindgen_object_drop_ref = function(arg0) { + takeObject(arg0); + }; + imports.wbg.__wbg_newwithu8clampedarrayandsh_87d2f0a48030f922 = function() { + return handleError(function(arg0, arg1, arg2, arg3) { + const ret = new ImageData(getClampedArrayU8FromWasm0(arg0, arg1), arg2 >>> 0, arg3 >>> 0); + return addHeapObject(ret); + }, arguments) + }; + imports.wbg.__wbindgen_throw = function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }; + if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) { + input = fetch(input); + } + const { + instance, + module + } = await load(await input, imports); + wasm = instance.exports; + init.__wbindgen_wasm_module = module; + return wasm; +} + +var arr; + +async function run() { + self.addEventListener('message', function(evt) { + try { + let length = evt.data.length; + let data = new Uint8Array(evt.data.sab.slice(0, length)); + let resultData = decode_qoi(data); + if (!arr) { + arr = new Uint8Array(evt.data.sabR); + } + let lengthR = resultData.data.length; + arr.set(resultData.data); + let img = { + colorSpace: resultData.colorSpace, + width: resultData.width, + height: resultData.height + }; + self.postMessage({ + result: 0, + img: img, + length: lengthR, + width: evt.data.width, + height: evt.data.height, + x: evt.data.x, + y: evt.data.y, + frame_id: evt.data.frame_id + }); + } catch (err) { + self.postMessage({ + result: 2, + error: err + }); + } + }, false); + + await init(); + + //Send message that worker is ready + self.postMessage({ + result: 1 + }) +} + +run(); diff --git a/core/decoders/qoi/qoi_viewer_bg.wasm b/core/decoders/qoi/qoi_viewer_bg.wasm new file mode 100644 index 00000000..1447352a Binary files /dev/null and b/core/decoders/qoi/qoi_viewer_bg.wasm differ diff --git a/core/decoders/raw.js b/core/decoders/raw.js index e8ea178e..fd2a9209 100644 --- a/core/decoders/raw.js +++ b/core/decoders/raw.js @@ -12,7 +12,7 @@ export default class RawDecoder { this._lines = 0; } - decodeRect(x, y, width, height, sock, display, depth) { + decodeRect(x, y, width, height, sock, display, depth, frame_id) { if ((width === 0) || (height === 0)) { return true; } @@ -54,7 +54,7 @@ export default class RawDecoder { data[i * 4 + 3] = 255; } - display.blitImage(x, curY, width, currHeight, data, index); + display.blitImage(x, curY, width, currHeight, data, index, frame_id); sock.rQskipBytes(currHeight * bytesPerLine); this._lines -= currHeight; if (this._lines > 0) { diff --git a/core/decoders/rre.js b/core/decoders/rre.js index 6219369d..0562fbc2 100644 --- a/core/decoders/rre.js +++ b/core/decoders/rre.js @@ -12,7 +12,7 @@ export default class RREDecoder { this._subrects = 0; } - decodeRect(x, y, width, height, sock, display, depth) { + decodeRect(x, y, width, height, sock, display, depth, frame_id) { if (this._subrects === 0) { if (sock.rQwait("RRE", 4 + 4)) { return false; @@ -34,7 +34,7 @@ export default class RREDecoder { let sy = sock.rQshift16(); let swidth = sock.rQshift16(); let sheight = sock.rQshift16(); - display.fillRect(x + sx, y + sy, swidth, sheight, color); + display.fillRect(x + sx, y + sy, swidth, sheight, color, frame_id); this._subrects--; } diff --git a/core/decoders/tight.js b/core/decoders/tight.js index 6e2799e3..dd111ece 100644 --- a/core/decoders/tight.js +++ b/core/decoders/tight.js @@ -12,12 +12,14 @@ import * as Log from '../util/logging.js'; import Inflator from "../inflator.js"; export default class TightDecoder { - constructor() { + constructor(display) { this._ctl = null; this._filter = null; this._numColors = 0; this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel) this._len = 0; + this._enableQOI = false; + this._displayGlobal = display; this._zlibs = []; for (let i = 0; i < 4; i++) { @@ -25,7 +27,14 @@ export default class TightDecoder { } } - decodeRect(x, y, width, height, sock, display, depth) { + enableQOI() { + if (!this._enableQOI) { + this._enableQOIWorkers(); + } + return this._enableQOI; //did it succeed + } + + decodeRect(x, y, width, height, sock, display, depth, frame_id) { if (this._ctl === null) { if (sock.rQwait("TIGHT compression-control", 1)) { return false; @@ -37,7 +46,7 @@ export default class TightDecoder { for (let i = 0; i < 4; i++) { if ((this._ctl >> i) & 1) { this._zlibs[i].reset(); - Log.Debug("Reset zlib stream " + i); + Log.Info("Reset zlib stream " + i); } } @@ -49,19 +58,22 @@ export default class TightDecoder { if (this._ctl === 0x08) { ret = this._fillRect(x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); } else if (this._ctl === 0x09) { ret = this._jpegRect(x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); } else if (this._ctl === 0x0A) { ret = this._pngRect(x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); } else if ((this._ctl & 0x08) == 0) { ret = this._basicRect(this._ctl, x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); } else if (this._ctl === 0x0B) { ret = this._webpRect(x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); + } else if (this._ctl === 0x0C) { + ret = this._qoiRect(x, y, width, height, + sock, display, depth, frame_id); } else { throw new Error("Illegal tight compression received (ctl: " + this._ctl + ")"); @@ -74,7 +86,7 @@ export default class TightDecoder { return ret; } - _fillRect(x, y, width, height, sock, display, depth) { + _fillRect(x, y, width, height, sock, display, depth, frame_id) { if (sock.rQwait("TIGHT", 3)) { return false; } @@ -83,39 +95,81 @@ export default class TightDecoder { const rQ = sock.rQ; display.fillRect(x, y, width, height, - [rQ[rQi], rQ[rQi + 1], rQ[rQi + 2]], false); + [rQ[rQi], rQ[rQi + 1], rQ[rQi + 2]], frame_id, false); sock.rQskipBytes(3); return true; } - _jpegRect(x, y, width, height, sock, display, depth) { + _jpegRect(x, y, width, height, sock, display, depth, frame_id) { let data = this._readData(sock); if (data === null) { return false; } - display.imageRect(x, y, width, height, "image/jpeg", data); + display.imageRect(x, y, width, height, "image/jpeg", data, frame_id); return true; } - _webpRect(x, y, width, height, sock, display, depth) { + _webpRect(x, y, width, height, sock, display, depth, frame_id) { let data = this._readData(sock); if (data === null) { return false; } - display.imageRect(x, y, width, height, "image/webp", data); + display.imageRect(x, y, width, height, "image/webp", data, frame_id); return true; } - _pngRect(x, y, width, height, sock, display, depth) { + _processRectQ() { + while (this._availableWorkers.length > 0 && this._qoiRects.length > 0) { + let i = this._availableWorkers.pop(); + let worker = this._workers[i]; + let rect = this._qoiRects.shift(); + this._arrs[i].set(rect.data); + worker.postMessage({ + length: rect.data.length, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + depth: rect.depth, + sab: this._sabs[i], + sabR: this._sabsR[i], + frame_id: rect.frame_id + }); + } + } + + _qoiRect(x, y, width, height, sock, display, depth, frame_id) { + let data = this._readData(sock); + if (data === null) { + return false; + } + + if (this._enableQOI) { + let dataClone = new Uint8Array(data); + let item = {x: x,y: y,width: width,height: height,data: dataClone,depth: depth, frame_id: frame_id}; + if (this._qoiRects.length < 1000) { + this._qoiRects.push(item); + this._processRectQ(); + } else { + Log.Warn("QOI queue exceeded limit."); + this._qoiRects.splice(0, 500); + } + + } + + return true; + } + + _pngRect(x, y, width, height, sock, display, depth, frame_id) { throw new Error("PNG received in standard Tight rect"); } - _basicRect(ctl, x, y, width, height, sock, display, depth) { + _basicRect(ctl, x, y, width, height, sock, display, depth, frame_id) { if (this._filter === null) { if (ctl & 0x4) { if (sock.rQwait("TIGHT", 1)) { @@ -136,15 +190,15 @@ export default class TightDecoder { switch (this._filter) { case 0: // CopyFilter ret = this._copyFilter(streamId, x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); break; case 1: // PaletteFilter ret = this._paletteFilter(streamId, x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); break; case 2: // GradientFilter ret = this._gradientFilter(streamId, x, y, width, height, - sock, display, depth); + sock, display, depth, frame_id); break; default: throw new Error("Illegal tight filter received (ctl: " + @@ -158,7 +212,7 @@ export default class TightDecoder { return ret; } - _copyFilter(streamId, x, y, width, height, sock, display, depth) { + _copyFilter(streamId, x, y, width, height, sock, display, depth, frame_id) { const uncompressedSize = width * height * 3; let data; @@ -191,12 +245,12 @@ export default class TightDecoder { rgbx[i + 3] = 255; // Alpha } - display.blitImage(x, y, width, height, rgbx, 0, false); + display.blitImage(x, y, width, height, rgbx, 0, frame_id, false); return true; } - _paletteFilter(streamId, x, y, width, height, sock, display, depth) { + _paletteFilter(streamId, x, y, width, height, sock, display, depth, frame_id) { if (this._numColors === 0) { if (sock.rQwait("TIGHT palette", 1)) { return false; @@ -244,9 +298,9 @@ export default class TightDecoder { // Convert indexed (palette based) image data to RGB if (this._numColors == 2) { - this._monoRect(x, y, width, height, data, this._palette, display); + this._monoRect(x, y, width, height, data, this._palette, display, frame_id); } else { - this._paletteRect(x, y, width, height, data, this._palette, display); + this._paletteRect(x, y, width, height, data, this._palette, display, frame_id); } this._numColors = 0; @@ -254,7 +308,7 @@ export default class TightDecoder { return true; } - _monoRect(x, y, width, height, data, palette, display) { + _monoRect(x, y, width, height, data, palette, display, frame_id) { // Convert indexed (palette based) image data to RGB // TODO: reduce number of calculations inside loop const dest = this._getScratchBuffer(width * height * 4); @@ -284,10 +338,10 @@ export default class TightDecoder { } } - display.blitImage(x, y, width, height, dest, 0, false); + display.blitImage(x, y, width, height, dest, 0, frame_id, false); } - _paletteRect(x, y, width, height, data, palette, display) { + _paletteRect(x, y, width, height, data, palette, display, frame_id) { // Convert indexed (palette based) image data to RGB const dest = this._getScratchBuffer(width * height * 4); const total = width * height * 4; @@ -299,10 +353,10 @@ export default class TightDecoder { dest[i + 3] = 255; } - display.blitImage(x, y, width, height, dest, 0, false); + display.blitImage(x, y, width, height, dest, 0, frame_id, false); } - _gradientFilter(streamId, x, y, width, height, sock, display, depth) { + _gradientFilter(streamId, x, y, width, height, sock, display, depth, frame_id) { throw new Error("Gradient filter not implemented"); } @@ -342,4 +396,63 @@ export default class TightDecoder { } return this._scratchBuffer; } + + _enableQOIWorkers() { + let sabTest = typeof SharedArrayBuffer; + if (sabTest !== 'undefined') { + this._enableQOI = true; + if ((window.navigator.hardwareConcurrency) && (window.navigator.hardwareConcurrency > 8)) { + this._threads = window.navigator.hardwareConcurrency; + } else { + this._threads = 8; + } + this._workerEnabled = false; + this._workers = []; + this._availableWorkers = []; + this._sabs = []; + this._sabsR = []; + this._arrs = []; + this._arrsR = []; + this._qoiRects = []; + this._rectQlooping = false; + for (let i = 0; i < this._threads; i++) { + this._workers.push(new Worker("/core/decoders/qoi/decoder.js")); + this._sabs.push(new SharedArrayBuffer(300000)); + this._sabsR.push(new SharedArrayBuffer(400000)); + this._arrs.push(new Uint8Array(this._sabs[i])); + this._arrsR.push(new Uint8ClampedArray(this._sabsR[i])); + this._workers[i].onmessage = (evt) => { + this._availableWorkers.push(i); + switch(evt.data.result) { + case 0: + let data = new Uint8ClampedArray(evt.data.length); + data.set(this._arrsR[i].slice(0, evt.data.length)); + let img = new ImageData(data, evt.data.img.width, evt.data.img.height, {colorSpace: evt.data.img.colorSpace}); + + this._displayGlobal.blitQoi( + evt.data.x, + evt.data.y, + evt.data.width, + evt.data.height, + img, + 0, + evt.data.frame_id, + false + ); + this._processRectQ(); + break; + case 1: + Log.Info("QOI Worker is now available."); + break; + case 2: + Log.Info("Error on worker: " + evt.error); + break; + } + }; + } + } else { + this._enableQOI = false; + Log.Warn("Enabling QOI Failed, client not compatible."); + } + } } diff --git a/core/decoders/tightpng.js b/core/decoders/tightpng.js index 82f492de..9ed28b8e 100644 --- a/core/decoders/tightpng.js +++ b/core/decoders/tightpng.js @@ -10,13 +10,13 @@ import TightDecoder from './tight.js'; export default class TightPNGDecoder extends TightDecoder { - _pngRect(x, y, width, height, sock, display, depth) { + _pngRect(x, y, width, height, sock, display, depth, frame_id) { let data = this._readData(sock); if (data === null) { return false; } - display.imageRect(x, y, width, height, "image/png", data); + display.imageRect(x, y, width, height, "image/png", data, frame_id); return true; } diff --git a/core/decoders/udp.js b/core/decoders/udp.js index 13650f79..f59e8119 100644 --- a/core/decoders/udp.js +++ b/core/decoders/udp.js @@ -22,7 +22,7 @@ export default class UDPDecoder { } } - decodeRect(x, y, width, height, data, display, depth) { + decodeRect(x, y, width, height, data, display, depth, frame_id) { let ctl = data[12]; ctl = ctl >> 4; diff --git a/core/display.js b/core/display.js index 96be93cb..be4aeaf4 100644 --- a/core/display.js +++ b/core/display.js @@ -9,12 +9,31 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; +import { isWindows } from './util/browser.js'; export default class Display { constructor(target) { - this._renderQ = []; // queue drawing actions for in-oder rendering - this._currentFrame = []; - this._nextFrame = []; + Log.Debug(">> Display.constructor"); + + /* + For performance reasons we use a multi dimensional array + 1st Dimension of Array Represents Frames, each element is a Frame + 2nd Dimension contains 4 elements + 0 - int, FrameID + 1 - int, Rect Count + 2 - Array of Rect objects + 3 - bool, is the frame complete + 4 - int, index of current rect (post-processing) + */ + this._asyncFrameQueue = []; + /* + Buffer for incoming frames. The larger the buffer the more time there is to collect, process, and order rects + but the more delay there is. May need to adjust this higher for lower power devices when UDP is complete. + Decoders that use WASM in parallel can also cause out of order rects + */ + this._maxAsyncFrameQueue = 3; + this._clearAsyncQueue(); + this._flushing = false; // the full frame buffer (logical canvas) size @@ -22,12 +41,7 @@ export default class Display { this._fbHeight = 0; this._renderMs = 0; - this._prevDrawStyle = ""; - - Log.Debug(">> Display.constructor"); - - // The visible canvas this._target = target; if (!this._target) { @@ -49,20 +63,22 @@ export default class Display { Log.Debug("User Agent: " + navigator.userAgent); - // performance metrics, try to calc a fps equivelant + // performance metrics this._flipCnt = 0; this._lastFlip = Date.now(); + this._droppedFrames = 0; + this._droppedRects = 0; + this._missingRectCnt = 0; setInterval(function() { let delta = Date.now() - this._lastFlip; if (delta > 0) { this._fps = (this._flipCnt / (delta / 1000)).toFixed(2); } + Log.Info('Dropped Frames: ' + this._droppedFrames + ' Dropped Rects: ' + this._droppedRects + ' Missing Rect Cnt: ' + this._missingRectCnt); this._flipCnt = 0; this._lastFlip = Date.now(); }.bind(this), 5000); - Log.Debug("<< Display.constructor"); - // ===== PROPERTIES ===== this._scale = 1.0; @@ -73,6 +89,11 @@ export default class Display { // ===== EVENT HANDLERS ===== this.onflush = () => { }; // A flush request has finished + + // Use requestAnimationFrame to write to canvas, to match display refresh rate + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + + Log.Debug("<< Display.constructor"); } // ===== PROPERTIES ===== @@ -149,8 +170,6 @@ export default class Display { return; } Log.Debug("viewportChange deltaX: " + deltaX + ", deltaY: " + deltaY); - - this.flip(); } viewportChangeSize(width, height) { @@ -186,8 +205,6 @@ export default class Display { // The position might need to be updated if we've grown this.viewportChangePos(0, 0); - this.flip(); - // Update the visible size of the target canvas this._rescale(this._scale); } @@ -243,54 +260,37 @@ export default class Display { } // rendering canvas - flip(fromQueue) { - if (!fromQueue) { - this._renderQPush({ - 'type': 'flip' - }); - } else { - for (let i = 0; i < this._currentFrame.length; i++) { - const a = this._currentFrame[i]; - switch (a.type) { - case 'copy': - this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, true); - break; - case 'fill': - this.fillRect(a.x, a.y, a.width, a.height, a.color, true); - break; - case 'blit': - this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, true); - break; - case 'img': - this.drawImage(a.img, a.x, a.y, a.width, a.height); - break; - } - } - this._flipCnt += 1; - } + flip(frame_id, rect_cnt) { + this._asyncRenderQPush({ + 'type': 'flip', + 'frame_id': frame_id, + 'rect_cnt': rect_cnt + }); } pending() { - return this._renderQ.length > 0; + //is the slot in the queue for the newest frame in use + return this._asyncFrameQueue[this._maxAsyncFrameQueue - 1][0] > 0; } flush() { - if (this._renderQ.length === 0) { - this.onflush(); - } else { - this._flushing = true; - } + //force oldest frame to render + this._asyncFrameComplete(0, true); + + //this in effect blocks more incoming frames until the oldest frame has been rendered to canvas (tcp only) + this._flushing = true; } - fillRect(x, y, width, height, color, fromQueue) { + fillRect(x, y, width, height, color, frame_id, fromQueue) { if (!fromQueue) { - this._renderQPush({ + this._asyncRenderQPush({ 'type': 'fill', 'x': x, 'y': y, 'width': width, 'height': height, - 'color': color + 'color': color, + 'frame_id': frame_id }); } else { this._setFillColor(color); @@ -298,9 +298,9 @@ export default class Display { } } - copyImage(oldX, oldY, newX, newY, w, h, fromQueue) { + copyImage(oldX, oldY, newX, newY, w, h, frame_id, fromQueue) { if (!fromQueue) { - this._renderQPush({ + this._asyncRenderQPush({ 'type': 'copy', 'oldX': oldX, 'oldY': oldY, @@ -308,6 +308,7 @@ export default class Display { 'y': newY, 'width': w, 'height': h, + 'frame_id': frame_id }); } else { // Due to this bug among others [1] we need to disable the image-smoothing to @@ -328,39 +329,40 @@ export default class Display { } } - imageRect(x, y, width, height, mime, arr) { + imageRect(x, y, width, height, mime, arr, frame_id) { /* The internal logic cannot handle empty images, so bail early */ if ((width === 0) || (height === 0)) { return; } - const img = new Image(); img.src = "data: " + mime + ";base64," + Base64.encode(arr); - this._renderQPush({ + this._asyncRenderQPush({ 'type': 'img', 'img': img, 'x': x, 'y': y, 'width': width, - 'height': height + 'height': height, + 'frame_id': frame_id }); } - blitImage(x, y, width, height, arr, offset, fromQueue) { + blitImage(x, y, width, height, arr, offset, frame_id, fromQueue) { if (!fromQueue) { // NB(directxman12): it's technically more performant here to use preallocated arrays, // but it's a lot of extra work for not a lot of payoff -- if we're using the render queue, // this probably isn't getting called *nearly* as much const newArr = new Uint8Array(width * height * 4); newArr.set(new Uint8Array(arr.buffer, 0, newArr.length)); - this._renderQPush({ + this._asyncRenderQPush({ 'type': 'blit', 'data': newArr, 'x': x, 'y': y, 'width': width, 'height': height, + 'frame_id': frame_id }); } else { // NB(directxman12): arr must be an Type Array view @@ -372,9 +374,25 @@ export default class Display { } } + blitQoi(x, y, width, height, arr, offset, frame_id, fromQueue) { + if (!fromQueue) { + this._asyncRenderQPush({ + 'type': 'blitQ', + 'data': arr, + 'x': x, + 'y': y, + 'width': width, + 'height': height, + 'frame_id': frame_id + }); + } else { + this._targetCtx.putImageData(arr, x, y); + } + } + drawImage(img, x, y, w, h) { try { - if (img.width != w || img.height != h) { + if (img.width != w || img.height != h) { this._targetCtx.drawImage(img, x, y, w, h); } else { this._targetCtx.drawImage(img, x, y); @@ -407,6 +425,144 @@ export default class Display { // ===== PRIVATE METHODS ===== + /* + Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding + */ + _asyncRenderQPush(rect) { + let frameIx = -1; + let oldestFrameID = Number.MAX_SAFE_INTEGER; + let newestFrameID = 0; + for (let i=0; i= 0) { + if (rect.type == "flip") { + //flip rect contains the rect count for the frame + this._asyncFrameQueue[frameIx][1] = rect.rect_cnt; + } + + if (this._asyncFrameQueue[frameIx][1] == this._asyncFrameQueue[frameIx][2].length) { + //frame is complete + this._asyncFrameComplete(frameIx); + } + } else { + if (rect.frame_id < oldestFrameID) { + //rect is older than any frame in the queue, drop it + this._droppedRects++; + return; + } else if (rect.frame_id > newestFrameID) { + //frame is newer than any frame in the queue, drop old frames + this._asyncFrameQueue.shift(); + let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0); + this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0 ]); + this._droppedFrames++; + } + } + + } + + /* + Clear the async frame buffer + */ + _clearAsyncQueue() { + this._droppedFrames += this._asyncFrameQueue.length; + + this._asyncFrameQueue = []; + for (let i=0; i { this._asyncFrameComplete(frameIx); }); + this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; + return; + } + currentFrameRectIx++; + } + } + this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; + this._asyncFrameQueue[frameIx][3] = true; + } + + /* + Push the oldest frame in the buffer to the canvas if it is marked ready + */ + _pushAsyncFrame() { + if (this._asyncFrameQueue[0][3]) { + let frame = this._asyncFrameQueue.shift()[2]; + this._asyncFrameQueue.push([ 0, 0, [], false, 0 ]); + + //render the selected frame + for (let i = 0; i < frame.length; i++) { + + const a = frame[i]; + switch (a.type) { + case 'copy': + this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, a.frame_id, true); + break; + case 'fill': + this.fillRect(a.x, a.y, a.width, a.height, a.color, a.frame_id, true); + break; + case 'blit': + this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'blitQ': + this.blitQoi(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'img': + this.drawImage(a.img, a.x, a.y, a.width, a.height); + break; + } + } + this._flipCnt += 1; + + if (this._flushing) { + this._flushing = false; + this.onflush(); + } + } + + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + } + _rescale(factor) { this._scale = factor; const vp = this._viewportLoc; @@ -445,56 +601,4 @@ export default class Display { this._prevDrawStyle = newStyle; } } - - _renderQPush(action) { - this._renderQ.push(action); - if (this._renderQ.length === 1) { - this._scanRenderQ(); - } - } - - _resumeRenderQ() { - this.removeEventListener('load', this._noVNCDisplay._resumeRenderQ); - this._noVNCDisplay._scanRenderQ(); - } - - _scanRenderQ() { - let ready = true; - let before = Date.now(); - while (ready && this._renderQ.length > 0) { - const a = this._renderQ[0]; - switch (a.type) { - case 'flip': - this._currentFrame = this._nextFrame; - this._nextFrame = []; - this.flip(true); - break; - case 'img': - if (a.img.complete) { - this._nextFrame.push(a); - } else { - a.img._noVNCDisplay = this; - a.img.addEventListener('load', this._resumeRenderQ); - // We need to wait for this image to 'load' - ready = false; - } - break; - default: - this._nextFrame.push(a); - break; - } - - if (ready) { - this._renderQ.shift(); - } - } - - if (this._renderQ.length === 0 && this._flushing) { - this._flushing = false; - this.onflush(); - } - - let elapsed = Date.now() - before; - this._renderMs += elapsed; - } } diff --git a/core/encodings.js b/core/encodings.js index c89e3730..af2f4493 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -53,6 +53,7 @@ export const encodings = { pseudoEncodingVideoScalingLevel9: -1987, pseudoEncodingVideoOutTimeLevel1: -1986, pseudoEncodingVideoOutTimeLevel100: -1887, + pseudoEncodingQOI: -1886, pseudoEncodingVMwareCursor: 0x574d5664, pseudoEncodingVMwareCursorPosition: 0x574d5666, diff --git a/core/rfb.js b/core/rfb.js index 8b2ff466..721358cd 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -43,6 +43,7 @@ const DEFAULT_BACKGROUND = 'rgb(40, 40, 40)'; var _videoQuality = 2; var _enableWebP = false; +var _enableQOI = false; // Minimum wait (ms) between two mouse moves const MOUSE_MOVE_DELAY = 17; @@ -139,6 +140,7 @@ export default class RFB extends EventTargetMixin { this._maxVideoResolutionY = 540; this._clipboardBinary = true; this._useUdp = true; + this._enableQOI = false; this.TransitConnectionStates = { Tcp: Symbol("tcp"), Udp: Symbol("udp"), @@ -173,12 +175,14 @@ export default class RFB extends EventTargetMixin { this._decoders = {}; this._FBU = { - rects: 0, + rects: 0, // current rect number x: 0, y: 0, width: 0, height: 0, encoding: null, + frame_id: 0, + rect_total: 0, //Total rects in frame }; // Mouse state @@ -248,15 +252,6 @@ export default class RFB extends EventTargetMixin { // initial cursor instead. this._cursorImage = RFB.cursors.none; - // populate decoder array with objects - this._decoders[encodings.encodingRaw] = new RawDecoder(); - this._decoders[encodings.encodingCopyRect] = new CopyRectDecoder(); - this._decoders[encodings.encodingRRE] = new RREDecoder(); - this._decoders[encodings.encodingHextile] = new HextileDecoder(); - this._decoders[encodings.encodingTight] = new TightDecoder(); - this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); - this._decoders[encodings.encodingUDP] = new UDPDecoder(); - // NB: nothing that needs explicit teardown should be done // before this point, since this can throw an exception try { @@ -267,6 +262,15 @@ export default class RFB extends EventTargetMixin { } this._display.onflush = this._onFlush.bind(this); + // populate decoder array with objects + this._decoders[encodings.encodingRaw] = new RawDecoder(); + this._decoders[encodings.encodingCopyRect] = new CopyRectDecoder(); + this._decoders[encodings.encodingRRE] = new RREDecoder(); + this._decoders[encodings.encodingHextile] = new HextileDecoder(); + this._decoders[encodings.encodingTight] = new TightDecoder(this._display); + this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); + this._decoders[encodings.encodingUDP] = new UDPDecoder(); + this._keyboard = new Keyboard(this._canvas, touchInput); this._keyboard.onkeyevent = this._handleKeyEvent.bind(this); @@ -481,6 +485,21 @@ export default class RFB extends EventTargetMixin { this._pendingApplyEncodingChanges = true; } + get enableQOI() { return this._enableQOI; } + set enableQOI(enabled) { + if(this._enableQOI === enabled) { + return; + } + if (enabled) { + if (!this._decoders[encodings.encodingTight].enableQOI()) { + //enabling qoi failed + return; + } + } + this._enableQOI = enabled; + this._pendingApplyEncodingChanges = true; + } + get antiAliasing() { return this._display.antiAliasing; } set antiAliasing(value) { this._display.antiAliasing = value; @@ -2520,12 +2539,8 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.encodingRaw); // Psuedo-encoding settings - var quality = 6; - var compression = 2; - var screensize = this._screenSize(false); encs.push(encodings.pseudoEncodingQualityLevel0 + this._qualityLevel); encs.push(encodings.pseudoEncodingCompressLevel0 + this._compressionLevel); - encs.push(encodings.pseudoEncodingDesktopSize); encs.push(encodings.pseudoEncodingLastRect); encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent); @@ -2537,6 +2552,9 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingExtendedClipboard); if (this._hasWebp()) encs.push(encodings.pseudoEncodingWEBP); + if (this._enableQOI) + encs.push(encodings.pseudoEncodingQOI); + // kasm settings; the server may be configured to ignore these encs.push(encodings.pseudoEncodingJpegVideoQualityLevel0 + this.jpegVideoQuality); @@ -3053,11 +3071,11 @@ export default class RFB extends EventTargetMixin { encoding: parseInt((data[8] << 24) + (data[9] << 16) + (data[10] << 8) + data[11], 10) }; - + switch (frame.encoding) { case encodings.pseudoEncodingLastRect: if (document.visibilityState !== "hidden") { - this._display.flip(); + this._display.flip(false); //TODO: UDP is now broken, flip needs rect count and frame number this._udpBuffer.clear(); } break; @@ -3158,6 +3176,9 @@ export default class RFB extends EventTargetMixin { this._sock.rQskipBytes(1); // Padding this._FBU.rects = this._sock.rQshift16(); + this._FBU.frame_id++; + this._FBU.rect_total = 0; + // Make sure the previous frame is fully rendered first // to avoid building up an excessive queue if (this._display.pending()) { @@ -3180,7 +3201,8 @@ export default class RFB extends EventTargetMixin { this._FBU.encoding = parseInt((hdr[8] << 24) + (hdr[9] << 16) + (hdr[10] << 8) + hdr[11], 10); } - + + if (!this._handleRect()) { return false; } @@ -3189,14 +3211,17 @@ export default class RFB extends EventTargetMixin { this._FBU.encoding = null; } - this._display.flip(); - + if (this._FBU.rect_total > 0) { + this._display.flip(this._FBU.frame_id, this._FBU.rect_total); + } + return true; // We finished this FBU } _handleRect() { switch (this._FBU.encoding) { case encodings.pseudoEncodingLastRect: + this._FBU.rect_total++; //only track rendered rects and last rect this._FBU.rects = 1; // Will be decreased when we return return true; @@ -3224,7 +3249,11 @@ export default class RFB extends EventTargetMixin { return this._handleExtendedDesktopSize(); default: - return this._handleDataRect(); + if (this._handleDataRect()) { + this._FBU.rect_total++; //only track rendered rects and last rect + return true; + } + return false; } } @@ -3524,9 +3553,9 @@ export default class RFB extends EventTargetMixin { return decoder.decodeRect(this._FBU.x, this._FBU.y, this._FBU.width, this._FBU.height, this._sock, this._display, - this._fbDepth); + this._fbDepth, this._FBU.frame_id); } catch (err) { - this._fail("Error decoding rect: " + err); + this._fail("Error decoding rect: " + err); return false; } } diff --git a/vnc.html b/vnc.html index 1ee56b00..2f071648 100644 --- a/vnc.html +++ b/vnc.html @@ -277,6 +277,12 @@ IME Input Mode +
  • + +