From 6c07136169830d70a1db2d67f7b955b6bc834d6f Mon Sep 17 00:00:00 2001 From: leedagee <61650578+leedagee@users.noreply.github.com> Date: Thu, 1 Aug 2024 01:31:47 +0800 Subject: [PATCH 01/13] Interrupt AltGr sequence detection on focus lost, fixes #1880 --- core/input/keyboard.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/input/keyboard.js b/core/input/keyboard.js index 68da2312..ff14de84 100644 --- a/core/input/keyboard.js +++ b/core/input/keyboard.js @@ -203,7 +203,7 @@ export default class Keyboard { if ((code === "ControlLeft") && browser.isWindows() && !("ControlLeft" in this._keyDownList)) { this._altGrArmed = true; - this._altGrTimeout = setTimeout(this._handleAltGrTimeout.bind(this), 100); + this._altGrTimeout = setTimeout(this._interruptAltGrSequence.bind(this), 100); this._altGrCtrlTime = e.timeStamp; return; } @@ -218,11 +218,7 @@ export default class Keyboard { // We can't get a release in the middle of an AltGr sequence, so // abort that detection - if (this._altGrArmed) { - this._altGrArmed = false; - clearTimeout(this._altGrTimeout); - this._sendKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); - } + this._interruptAltGrSequence(); // See comment in _handleKeyDown() if ((browser.isMac() || browser.isIOS()) && (code === 'CapsLock')) { @@ -249,14 +245,20 @@ export default class Keyboard { } } - _handleAltGrTimeout() { - this._altGrArmed = false; - clearTimeout(this._altGrTimeout); - this._sendKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); + _interruptAltGrSequence() { + if (this._altGrArmed) { + this._altGrArmed = false; + clearTimeout(this._altGrTimeout); + this._sendKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); + } } _allKeysUp() { Log.Debug(">> Keyboard.allKeysUp"); + + // Prevent control key being processed after losing focus. + this._interruptAltGrSequence(); + for (let code in this._keyDownList) { this._sendKeyEvent(this._keyDownList[code], code, false); } From bc31e4e8a274c0218de010c0d37b5e684e739ac4 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 5 Aug 2024 15:44:07 +0200 Subject: [PATCH 02/13] Stop creating sinon sandbox early sinon might not be loaded at this point, which can cause tests to fail. We could create the sandbox in one of the hooks instead, but let's remove the sandbox completely to stay consistent with our other tests. --- tests/test.webutil.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test.webutil.js b/tests/test.webutil.js index df8227ae..11d09309 100644 --- a/tests/test.webutil.js +++ b/tests/test.webutil.js @@ -182,16 +182,15 @@ describe('WebUtil', function () { window.chrome = chrome; }); - const csSandbox = sinon.createSandbox(); - beforeEach(function () { settings = {}; - csSandbox.spy(window.chrome.storage.sync, 'set'); - csSandbox.spy(window.chrome.storage.sync, 'remove'); + sinon.spy(window.chrome.storage.sync, 'set'); + sinon.spy(window.chrome.storage.sync, 'remove'); return WebUtil.initSettings(); }); afterEach(function () { - csSandbox.restore(); + window.chrome.storage.sync.set.restore(); + window.chrome.storage.sync.remove.restore(); }); describe('writeSetting', function () { From 1b2fe3321bface82604f2c5034cbc2ad8396560a Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 5 Aug 2024 14:23:54 +0200 Subject: [PATCH 03/13] Manually load sinon and chai karma-sinon-chai is not compatible with Chai 5+, and Karma is no longer being updated. Load sinon and chai manually instead, until we can have a long term plan in place. --- eslint.config.mjs | 2 +- karma.conf.js | 11 +++++++++-- package.json | 1 - tests/assertions.js | 9 +++++++++ tests/test.base64.js | 2 -- tests/test.browser.js | 2 -- tests/test.copyrect.js | 2 -- tests/test.deflator.js | 2 -- tests/test.display.js | 2 -- tests/test.gesturehandler.js | 2 -- tests/test.helper.js | 2 -- tests/test.hextile.js | 2 -- tests/test.inflator.js | 2 -- tests/test.int.js | 2 -- tests/test.jpeg.js | 2 -- tests/test.keyboard.js | 2 -- tests/test.localization.js | 1 - tests/test.raw.js | 2 -- tests/test.rfb.js | 2 -- tests/test.rre.js | 2 -- tests/test.tight.js | 2 -- tests/test.tightpng.js | 2 -- tests/test.util.js | 2 -- tests/test.websock.js | 2 -- tests/test.webutil.js | 2 -- tests/test.zrle.js | 2 -- 26 files changed, 19 insertions(+), 47 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index c88e7b75..13b1a32a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -79,7 +79,7 @@ export default [ ...globals.node, ...globals.mocha, sinon: false, - chai: false, + expect: false, } }, rules: { diff --git a/karma.conf.js b/karma.conf.js index 1ea17475..54380ebd 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -27,15 +27,22 @@ module.exports = (config) => { // frameworks to use // available frameworks: https://npmjs.org/browse/keyword/karma-adapter - frameworks: ['mocha', 'sinon-chai'], + frameworks: ['mocha'], - // list of files / patterns to load in the browser (loaded in order) + // list of files / patterns to load in the browser files: [ + // node modules + { pattern: 'node_modules/chai/**', included: false }, + { pattern: 'node_modules/sinon/**', included: false }, + { pattern: 'node_modules/sinon-chai/**', included: false }, + // modules to test { pattern: 'app/localization.js', included: false, type: 'module' }, { pattern: 'app/webutil.js', included: false, type: 'module' }, { pattern: 'core/**/*.js', included: false, type: 'module' }, { pattern: 'vendor/pako/**/*.js', included: false, type: 'module' }, + // tests { pattern: 'tests/test.*.js', type: 'module' }, + // test support files { pattern: 'tests/fake.*.js', included: false, type: 'module' }, { pattern: 'tests/assertions.js', type: 'module' }, ], diff --git a/package.json b/package.json index 9fa8c312..e28850a8 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "karma-mocha-reporter": "latest", "karma-safari-launcher": "latest", "karma-script-launcher": "latest", - "karma-sinon-chai": "latest", "mocha": "latest", "node-getopt": "latest", "po2json": "latest", diff --git a/tests/assertions.js b/tests/assertions.js index 739f6375..a7012271 100644 --- a/tests/assertions.js +++ b/tests/assertions.js @@ -1,3 +1,12 @@ +import * as chai from '../node_modules/chai/chai.js'; +import sinon from '../node_modules/sinon/pkg/sinon-esm.js'; +import sinonChai from '../node_modules/sinon-chai/lib/sinon-chai.js'; + +window.expect = chai.expect; + +window.sinon = sinon; +chai.use(sinonChai); + // noVNC specific assertions chai.use(function (_chai, utils) { function _equal(a, b) { diff --git a/tests/test.base64.js b/tests/test.base64.js index 04bd207b..e5644dcd 100644 --- a/tests/test.base64.js +++ b/tests/test.base64.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Base64 from '../core/base64.js'; describe('Base64 Tools', function () { diff --git a/tests/test.browser.js b/tests/test.browser.js index 1beeb48d..692cc23b 100644 --- a/tests/test.browser.js +++ b/tests/test.browser.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import { isMac, isWindows, isIOS, isAndroid, isChromeOS, isSafari, isFirefox, isChrome, isChromium, isOpera, isEdge, isGecko, isWebKit, isBlink } from '../core/util/browser.js'; diff --git a/tests/test.copyrect.js b/tests/test.copyrect.js index a10cddce..60c39528 100644 --- a/tests/test.copyrect.js +++ b/tests/test.copyrect.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.deflator.js b/tests/test.deflator.js index a7e972ec..b565b907 100644 --- a/tests/test.deflator.js +++ b/tests/test.deflator.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import { inflateInit, inflate } from "../vendor/pako/lib/zlib/inflate.js"; import ZStream from "../vendor/pako/lib/zlib/zstream.js"; import Deflator from "../core/deflator.js"; diff --git a/tests/test.display.js b/tests/test.display.js index e6c0406f..d2c51793 100644 --- a/tests/test.display.js +++ b/tests/test.display.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Base64 from '../core/base64.js'; import Display from '../core/display.js'; diff --git a/tests/test.gesturehandler.js b/tests/test.gesturehandler.js index 73356be3..d2e27ed2 100644 --- a/tests/test.gesturehandler.js +++ b/tests/test.gesturehandler.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import EventTargetMixin from '../core/util/eventtarget.js'; import GestureHandler from '../core/input/gesturehandler.js'; diff --git a/tests/test.helper.js b/tests/test.helper.js index 9995973f..2c8720c7 100644 --- a/tests/test.helper.js +++ b/tests/test.helper.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import keysyms from '../core/input/keysymdef.js'; import * as KeyboardUtil from "../core/input/util.js"; diff --git a/tests/test.hextile.js b/tests/test.hextile.js index cbe6f7b5..f788fd4d 100644 --- a/tests/test.hextile.js +++ b/tests/test.hextile.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.inflator.js b/tests/test.inflator.js index 304e7a0f..11a02f2f 100644 --- a/tests/test.inflator.js +++ b/tests/test.inflator.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import { deflateInit, deflate, Z_FULL_FLUSH } from "../vendor/pako/lib/zlib/deflate.js"; import ZStream from "../vendor/pako/lib/zlib/zstream.js"; import Inflator from "../core/inflator.js"; diff --git a/tests/test.int.js b/tests/test.int.js index 084d68ab..378ebd58 100644 --- a/tests/test.int.js +++ b/tests/test.int.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import { toUnsigned32bit, toSigned32bit } from '../core/util/int.js'; describe('Integer casting', function () { diff --git a/tests/test.jpeg.js b/tests/test.jpeg.js index 8dee4891..5cc153f9 100644 --- a/tests/test.jpeg.js +++ b/tests/test.jpeg.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.keyboard.js b/tests/test.keyboard.js index efc84c30..135c5981 100644 --- a/tests/test.keyboard.js +++ b/tests/test.keyboard.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Keyboard from '../core/input/keyboard.js'; describe('Key Event Handling', function () { diff --git a/tests/test.localization.js b/tests/test.localization.js index 916ff846..a1cb4547 100644 --- a/tests/test.localization.js +++ b/tests/test.localization.js @@ -1,4 +1,3 @@ -const expect = chai.expect; import _, { Localizer, l10n } from '../app/localization.js'; describe('Localization', function () { diff --git a/tests/test.raw.js b/tests/test.raw.js index 4a634ccd..19b2377f 100644 --- a/tests/test.raw.js +++ b/tests/test.raw.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 62b80ca3..2be3bfbf 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import RFB from '../core/rfb.js'; import Websock from '../core/websock.js'; import ZStream from "../vendor/pako/lib/zlib/zstream.js"; diff --git a/tests/test.rre.js b/tests/test.rre.js index c55d7f39..7b5f73d0 100644 --- a/tests/test.rre.js +++ b/tests/test.rre.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.tight.js b/tests/test.tight.js index 141d7b6e..3d6b555d 100644 --- a/tests/test.tight.js +++ b/tests/test.tight.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.tightpng.js b/tests/test.tightpng.js index 02c66d93..e7edc8fa 100644 --- a/tests/test.tightpng.js +++ b/tests/test.tightpng.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; diff --git a/tests/test.util.js b/tests/test.util.js index cd61f248..eb724095 100644 --- a/tests/test.util.js +++ b/tests/test.util.js @@ -1,6 +1,4 @@ /* eslint-disable no-console */ -const expect = chai.expect; - import * as Log from '../core/util/logging.js'; import { encodeUTF8, decodeUTF8 } from '../core/util/strings.js'; diff --git a/tests/test.websock.js b/tests/test.websock.js index dc361b74..53145b36 100644 --- a/tests/test.websock.js +++ b/tests/test.websock.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import FakeWebSocket from './fake.websocket.js'; diff --git a/tests/test.webutil.js b/tests/test.webutil.js index 11d09309..9151a060 100644 --- a/tests/test.webutil.js +++ b/tests/test.webutil.js @@ -1,7 +1,5 @@ /* jshint expr: true */ -const expect = chai.expect; - import * as WebUtil from '../app/webutil.js'; describe('WebUtil', function () { diff --git a/tests/test.zrle.js b/tests/test.zrle.js index be046409..f7c6089d 100644 --- a/tests/test.zrle.js +++ b/tests/test.zrle.js @@ -1,5 +1,3 @@ -const expect = chai.expect; - import Websock from '../core/websock.js'; import Display from '../core/display.js'; From 06f14a5cd3c188c63ad2060b7532a914c0d8f7f6 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 5 Aug 2024 16:31:59 +0200 Subject: [PATCH 04/13] Add test for AltGr abort on blur --- tests/test.keyboard.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test.keyboard.js b/tests/test.keyboard.js index 135c5981..11c8b6eb 100644 --- a/tests/test.keyboard.js +++ b/tests/test.keyboard.js @@ -478,6 +478,22 @@ describe('Key Event Handling', function () { expect(kbd.onkeyevent).to.not.have.been.called; }); + it('should release ControlLeft on blur', function () { + const kbd = new Keyboard(document); + kbd.onkeyevent = sinon.spy(); + kbd._handleKeyDown(keyevent('keydown', {code: 'ControlLeft', key: 'Control', location: 1})); + expect(kbd.onkeyevent).to.not.have.been.called; + kbd._allKeysUp(); + expect(kbd.onkeyevent).to.have.been.calledTwice; + expect(kbd.onkeyevent.firstCall).to.have.been.calledWith(0xffe3, "ControlLeft", true); + expect(kbd.onkeyevent.secondCall).to.have.been.calledWith(0xffe3, "ControlLeft", false); + + // Check that the timer is properly dead + kbd.onkeyevent.resetHistory(); + this.clock.tick(100); + expect(kbd.onkeyevent).to.not.have.been.called; + }); + it('should generate AltGraph for quick Ctrl+Alt sequence', function () { const kbd = new Keyboard(document); kbd.onkeyevent = sinon.spy(); From 074fa1a40f6e3f65bd61109de3f404796e266524 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 8 Aug 2024 14:40:04 +0200 Subject: [PATCH 05/13] Let browser construct URL string for us Likely a lot safer for corner cases than us trying to figure this out ourselves. --- app/ui.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/ui.js b/app/ui.js index f27dfe28..1a9571dc 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1033,16 +1033,17 @@ const UI = { let url; - url = UI.getSetting('encrypt') ? 'wss' : 'ws'; + url = new URL("https://" + host); - url += '://' + host; + url.protocol = UI.getSetting('encrypt') ? 'wss:' : 'ws:'; if (port) { - url += ':' + port; + url.port = port; } - url += '/' + path; + url.pathname = '/' + path; try { - UI.rfb = new RFB(document.getElementById('noVNC_container'), url, + UI.rfb = new RFB(document.getElementById('noVNC_container'), + url.href, { shared: UI.getSetting('shared'), repeaterID: UI.getSetting('repeaterID'), credentials: { password: password } }); From 96c76f7709037956e760ae5f58dbde1bc1308bba Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 8 Aug 2024 14:53:42 +0200 Subject: [PATCH 06/13] Allow relative WebSocket URLs This can be very useful if you have multiple instances of noVNC, and you want to redirect them to different VNC servers. The new default settings will have the same behaviour as before for systems where noVNC is deployed in the root web folder. --- app/ui.js | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/app/ui.js b/app/ui.js index 1a9571dc..90dc7189 100644 --- a/app/ui.js +++ b/app/ui.js @@ -158,20 +158,7 @@ const UI = { UI.initSetting('logging', 'warn'); UI.updateLogging(); - // if port == 80 (or 443) then it won't be present and should be - // set manually - let port = window.location.port; - if (!port) { - if (window.location.protocol.substring(0, 5) == 'https') { - port = 443; - } else if (window.location.protocol.substring(0, 4) == 'http') { - port = 80; - } - } - /* Populate the controls if defaults are provided in the URL */ - UI.initSetting('host', window.location.hostname); - UI.initSetting('port', port); UI.initSetting('encrypt', (window.location.protocol === "https:")); UI.initSetting('view_clip', false); UI.initSetting('resize', 'off'); @@ -1021,25 +1008,27 @@ const UI = { UI.hideStatus(); - if (!host) { - Log.Error("Can't connect when host is: " + host); - UI.showStatus(_("Must set host"), 'error'); - return; - } - UI.closeConnectPanel(); UI.updateVisualState('connecting'); let url; - url = new URL("https://" + host); + if (host) { + url = new URL("https://" + host); - url.protocol = UI.getSetting('encrypt') ? 'wss:' : 'ws:'; - if (port) { - url.port = port; + url.protocol = UI.getSetting('encrypt') ? 'wss:' : 'ws:'; + if (port) { + url.port = port; + } + url.pathname = '/' + path; + } else { + // Current (May 2024) browsers support relative WebSocket + // URLs natively, but we need to support older browsers for + // some time. + url = new URL(path, location.href); + url.protocol = (window.location.protocol === "https:") ? 'wss:' : 'ws:'; } - url.pathname = '/' + path; try { UI.rfb = new RFB(document.getElementById('noVNC_container'), From c6c8e5e51329001f778a72607bc68432b9d359f8 Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Thu, 15 Aug 2024 09:39:05 -0700 Subject: [PATCH 07/13] Add Zlib encoding --- README.md | 2 +- core/decoders/zlib.js | 51 ++++++++++++++++++++++++++ core/encodings.js | 2 ++ core/rfb.js | 3 ++ tests/test.zlib.js | 84 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 core/decoders/zlib.js create mode 100644 tests/test.zlib.js diff --git a/README.md b/README.md index b95d15e6..30510858 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ profits such as: RSA-AES, Tight, VeNCrypt Plain, XVP, Apple's Diffie-Hellman, UltraVNC's MSLogonII * Supported VNC encodings: raw, copyrect, rre, hextile, tight, tightPNG, - ZRLE, JPEG + ZRLE, JPEG, Zlib * Supports scaling, clipping and resizing the desktop * Local cursor rendering * Clipboard copy/paste with full Unicode support diff --git a/core/decoders/zlib.js b/core/decoders/zlib.js new file mode 100644 index 00000000..d1e5d5c9 --- /dev/null +++ b/core/decoders/zlib.js @@ -0,0 +1,51 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2024 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +import Inflator from "../inflator.js"; + +export default class ZlibDecoder { + constructor() { + this._zlib = new Inflator(); + this._length = 0; + } + + decodeRect(x, y, width, height, sock, display, depth) { + if ((width === 0) || (height === 0)) { + return true; + } + + if (this._length === 0) { + if (sock.rQwait("ZLIB", 4)) { + return false; + } + + this._length = sock.rQshift32(); + } + + if (sock.rQwait("ZLIB", this._length)) { + return false; + } + + let data = new Uint8Array(sock.rQshiftBytes(this._length, false)); + this._length = 0; + + this._zlib.setInput(data); + data = this._zlib.inflate(width * height * 4); + this._zlib.setInput(null); + + // Max sure the image is fully opaque + for (let i = 0; i < width * height; i++) { + data[i * 4 + 3] = 255; + } + + display.blitImage(x, y, width, height, data, 0); + + return true; + } +} diff --git a/core/encodings.js b/core/encodings.js index 1a79989d..d8053619 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -11,6 +11,7 @@ export const encodings = { encodingCopyRect: 1, encodingRRE: 2, encodingHextile: 5, + encodingZlib: 6, encodingTight: 7, encodingZRLE: 16, encodingTightPNG: -260, @@ -40,6 +41,7 @@ export function encodingName(num) { case encodings.encodingCopyRect: return "CopyRect"; case encodings.encodingRRE: return "RRE"; case encodings.encodingHextile: return "Hextile"; + case encodings.encodingZlib: return "Zlib"; case encodings.encodingTight: return "Tight"; case encodings.encodingZRLE: return "ZRLE"; case encodings.encodingTightPNG: return "TightPNG"; diff --git a/core/rfb.js b/core/rfb.js index f2deb0e7..0bd2b07e 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -31,6 +31,7 @@ import RawDecoder from "./decoders/raw.js"; import CopyRectDecoder from "./decoders/copyrect.js"; import RREDecoder from "./decoders/rre.js"; import HextileDecoder from "./decoders/hextile.js"; +import ZlibDecoder from './decoders/zlib.js'; import TightDecoder from "./decoders/tight.js"; import TightPNGDecoder from "./decoders/tightpng.js"; import ZRLEDecoder from "./decoders/zrle.js"; @@ -244,6 +245,7 @@ export default class RFB extends EventTargetMixin { this._decoders[encodings.encodingCopyRect] = new CopyRectDecoder(); this._decoders[encodings.encodingRRE] = new RREDecoder(); this._decoders[encodings.encodingHextile] = new HextileDecoder(); + this._decoders[encodings.encodingZlib] = new ZlibDecoder(); this._decoders[encodings.encodingTight] = new TightDecoder(); this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); this._decoders[encodings.encodingZRLE] = new ZRLEDecoder(); @@ -2121,6 +2123,7 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.encodingJPEG); encs.push(encodings.encodingHextile); encs.push(encodings.encodingRRE); + encs.push(encodings.encodingZlib); } encs.push(encodings.encodingRaw); diff --git a/tests/test.zlib.js b/tests/test.zlib.js new file mode 100644 index 00000000..bc72137e --- /dev/null +++ b/tests/test.zlib.js @@ -0,0 +1,84 @@ +import Websock from '../core/websock.js'; +import Display from '../core/display.js'; + +import ZlibDecoder from '../core/decoders/zlib.js'; + +import FakeWebSocket from './fake.websocket.js'; + +function testDecodeRect(decoder, x, y, width, height, data, display, depth) { + let sock; + let done = false; + + sock = new Websock; + sock.open("ws://example.com"); + + sock.on('message', () => { + done = decoder.decodeRect(x, y, width, height, sock, display, depth); + }); + + // Empty messages are filtered at multiple layers, so we need to + // do a direct call + if (data.length === 0) { + done = decoder.decodeRect(x, y, width, height, sock, display, depth); + } else { + sock._websocket._receiveData(new Uint8Array(data)); + } + + display.flip(); + + return done; +} + +describe('Zlib Decoder', function () { + let decoder; + let display; + + before(FakeWebSocket.replace); + after(FakeWebSocket.restore); + + beforeEach(function () { + decoder = new ZlibDecoder(); + display = new Display(document.createElement('canvas')); + display.resize(4, 4); + }); + + it('should handle the Zlib encoding', function () { + let done; + + let zlibData = new Uint8Array([ + 0x00, 0x00, 0x00, 0x23, /* length */ + 0x78, 0x01, 0xfa, 0xcf, 0x00, 0x04, 0xff, 0x61, 0x04, 0x90, 0x01, 0x41, 0x50, 0xc1, 0xff, 0x0c, + 0xef, 0x40, 0x02, 0xef, 0xfe, 0x33, 0xac, 0x02, 0xe2, 0xd5, 0x40, 0x8c, 0xce, 0x07, 0x00, 0x00, + 0x00, 0xff, 0xff, + ]); + done = testDecodeRect(decoder, 0, 0, 4, 4, zlibData, display, 24); + expect(done).to.be.true; + + let targetData = new Uint8ClampedArray([ + 0xff, 0x00, 0x00, 255, 0x00, 0xff, 0x00, 255, 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255, + 0x00, 0xff, 0x00, 255, 0xff, 0x00, 0x00, 255, 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255, + 0xee, 0x00, 0xff, 255, 0x00, 0xee, 0xff, 255, 0xaa, 0xee, 0xff, 255, 0xab, 0xee, 0xff, 255, + 0xee, 0x00, 0xff, 255, 0x00, 0xee, 0xff, 255, 0xaa, 0xee, 0xff, 255, 0xab, 0xee, 0xff, 255 + ]); + + expect(display).to.have.displayed(targetData); + }); + + it('should handle empty rects', function () { + display.fillRect(0, 0, 4, 4, [0x00, 0x00, 0xff]); + display.fillRect(2, 0, 2, 2, [0x00, 0xff, 0x00]); + display.fillRect(0, 2, 2, 2, [0x00, 0xff, 0x00]); + + let done = testDecodeRect(decoder, 1, 2, 0, 0, [], display, 24); + + let targetData = new Uint8Array([ + 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255, 0x00, 0xff, 0x00, 255, 0x00, 0xff, 0x00, 255, + 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255, 0x00, 0xff, 0x00, 255, 0x00, 0xff, 0x00, 255, + 0x00, 0xff, 0x00, 255, 0x00, 0xff, 0x00, 255, 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255, + 0x00, 0xff, 0x00, 255, 0x00, 0xff, 0x00, 255, 0x00, 0x00, 0xff, 255, 0x00, 0x00, 0xff, 255 + ]); + + expect(done).to.be.true; + expect(display).to.have.displayed(targetData); + }); +}); From d106b7a6bba12feec81d475586157be0193a94f6 Mon Sep 17 00:00:00 2001 From: Andri Yngvason Date: Sat, 29 Jun 2024 13:41:50 +0000 Subject: [PATCH 08/13] Add H.264 decoder This adds an H.264 decoder based on WebCodecs. --- core/decoders/h264.js | 321 ++++++++++++++++++++++++++++++++++++++++++ core/display.js | 53 ++++++- core/encodings.js | 2 + core/rfb.js | 7 +- core/util/browser.js | 20 +++ 5 files changed, 399 insertions(+), 4 deletions(-) create mode 100644 core/decoders/h264.js diff --git a/core/decoders/h264.js b/core/decoders/h264.js new file mode 100644 index 00000000..db144fcd --- /dev/null +++ b/core/decoders/h264.js @@ -0,0 +1,321 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2024 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +import * as Log from '../util/logging.js'; + +class H264Parser { + constructor(data) { + this._data = data; + this._index = 0; + this.profileIdc = null; + this.constraintSet = null; + this.levelIdc = null; + } + + _getStartSequenceLen(index) { + let data = this._data; + if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 0 && data[index + 3] == 1) { + return 4; + } + if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 1) { + return 3; + } + return 0; + } + + _indexOfNextNalUnit(index) { + let data = this._data; + for (let i = index; i < data.length; ++i) { + if (this._getStartSequenceLen(i) != 0) { + return i; + } + } + return -1; + } + + _parseSps(index) { + this.profileIdc = this._data[index]; + this.constraintSet = this._data[index + 1]; + this.levelIdc = this._data[index + 2]; + } + + _parseNalUnit(index) { + const firstByte = this._data[index]; + if (firstByte & 0x80) { + throw new Error('H264 parsing sanity check failed, forbidden zero bit is set'); + } + const unitType = firstByte & 0x1f; + + switch (unitType) { + case 1: // coded slice, non-idr + return { slice: true }; + case 5: // coded slice, idr + return { slice: true, key: true }; + case 6: // sei + return {}; + case 7: // sps + this._parseSps(index + 1); + return {}; + case 8: // pps + return {}; + default: + Log.Warn("Unhandled unit type: ", unitType); + break; + } + return {}; + } + + parse() { + const startIndex = this._index; + let isKey = false; + + while (this._index < this._data.length) { + const startSequenceLen = this._getStartSequenceLen(this._index); + if (startSequenceLen == 0) { + throw new Error('Invalid start sequence in bit stream'); + } + + const { slice, key } = this._parseNalUnit(this._index + startSequenceLen); + + let nextIndex = this._indexOfNextNalUnit(this._index + startSequenceLen); + if (nextIndex == -1) { + this._index = this._data.length; + } else { + this._index = nextIndex; + } + + if (key) { + isKey = true; + } + if (slice) { + break; + } + } + + if (startIndex === this._index) { + return null; + } + + return { + frame: this._data.subarray(startIndex, this._index), + key: isKey, + }; + } +} + +class H264Context { + constructor(width, height) { + this.lastUsed = 0; + this._width = width; + this._height = height; + this._profileIdc = null; + this._constraintSet = null; + this._levelIdc = null; + this._decoder = null; + this._pendingFrames = []; + } + + _handleFrame(frame) { + let pending = this._pendingFrames.shift(); + if (pending === undefined) { + throw new Error("Pending frame queue empty when receiving frame from decoder"); + } + + if (pending.timestamp != frame.timestamp) { + throw new Error("Video frame timestamp mismatch. Expected " + + frame.timestamp + " but but got " + pending.timestamp); + } + + pending.frame = frame; + pending.ready = true; + pending.resolve(); + + if (!pending.keep) { + frame.close(); + } + } + + _handleError(e) { + throw new Error("Failed to decode frame: " + e.message); + } + + _configureDecoder(profileIdc, constraintSet, levelIdc) { + if (this._decoder === null || this._decoder.state === 'closed') { + this._decoder = new VideoDecoder({ + output: frame => this._handleFrame(frame), + error: e => this._handleError(e), + }); + } + const codec = 'avc1.' + + profileIdc.toString(16).padStart(2, '0') + + constraintSet.toString(16).padStart(2, '0') + + levelIdc.toString(16).padStart(2, '0'); + this._decoder.configure({ + codec: codec, + codedWidth: this._width, + codedHeight: this._height, + optimizeForLatency: true, + }); + } + + _preparePendingFrame(timestamp) { + let pending = { + timestamp: timestamp, + promise: null, + resolve: null, + frame: null, + ready: false, + keep: false, + }; + pending.promise = new Promise((resolve) => { + pending.resolve = resolve; + }); + this._pendingFrames.push(pending); + + return pending; + } + + decode(payload) { + let parser = new H264Parser(payload); + let result = null; + + // Ideally, this timestamp should come from the server, but we'll just + // approximate it instead. + let timestamp = Math.round(window.performance.now() * 1e3); + + while (true) { + let encodedFrame = parser.parse(); + if (encodedFrame === null) { + break; + } + + if (parser.profileIdc !== null) { + self._profileIdc = parser.profileIdc; + self._constraintSet = parser.constraintSet; + self._levelIdc = parser.levelIdc; + } + + if (this._decoder === null || this._decoder.state !== 'configured') { + if (!encodedFrame.key) { + Log.Warn("Missing key frame. Can't decode until one arrives"); + continue; + } + if (self._profileIdc === null) { + Log.Warn('Cannot config decoder. Have not received SPS and PPS yet.'); + continue; + } + this._configureDecoder(self._profileIdc, self._constraintSet, + self._levelIdc); + } + + result = this._preparePendingFrame(timestamp); + + const chunk = new EncodedVideoChunk({ + timestamp: timestamp, + type: encodedFrame.key ? 'key' : 'delta', + data: encodedFrame.frame, + }); + + try { + this._decoder.decode(chunk); + } catch (e) { + Log.Warn("Failed to decode:", e); + } + } + + // We only keep last frame of each payload + if (result !== null) { + result.keep = true; + } + + return result; + } +} + +export default class H264Decoder { + constructor() { + this._tick = 0; + this._contexts = {}; + } + + _contextId(x, y, width, height) { + return [x, y, width, height].join(','); + } + + _findOldestContextId() { + let oldestTick = Number.MAX_VALUE; + let oldestKey = undefined; + for (const [key, value] of Object.entries(this._contexts)) { + if (value.lastUsed < oldestTick) { + oldestTick = value.lastUsed; + oldestKey = key; + } + } + return oldestKey; + } + + _createContext(x, y, width, height) { + const maxContexts = 64; + if (Object.keys(this._contexts).length >= maxContexts) { + let oldestContextId = this._findOldestContextId(); + delete this._contexts[oldestContextId]; + } + let context = new H264Context(width, height); + this._contexts[this._contextId(x, y, width, height)] = context; + return context; + } + + _getContext(x, y, width, height) { + let context = this._contexts[this._contextId(x, y, width, height)]; + return context !== undefined ? context : this._createContext(x, y, width, height); + } + + _resetContext(x, y, width, height) { + delete this._contexts[this._contextId(x, y, width, height)]; + } + + _resetAllContexts() { + this._contexts = {}; + } + + decodeRect(x, y, width, height, sock, display, depth) { + const resetContextFlag = 1; + const resetAllContextsFlag = 2; + + if (sock.rQwait("h264 header", 8)) { + return false; + } + + const length = sock.rQshift32(); + const flags = sock.rQshift32(); + + if (sock.rQwait("h264 payload", length, 8)) { + return false; + } + + if (flags & resetAllContextsFlag) { + this._resetAllContexts(); + } else if (flags & resetContextFlag) { + this._resetContext(x, y, width, height); + } + + let context = this._getContext(x, y, width, height); + context.lastUsed = this._tick++; + + if (length !== 0) { + let payload = sock.rQshiftBytes(length, false); + let frame = context.decode(payload); + if (frame !== null) { + display.videoFrame(x, y, width, height, frame); + } + } + + return true; + } +} diff --git a/core/display.js b/core/display.js index fcd62699..bc0bf219 100644 --- a/core/display.js +++ b/core/display.js @@ -380,6 +380,17 @@ export default class Display { }); } + videoFrame(x, y, width, height, frame) { + this._renderQPush({ + 'type': 'frame', + 'frame': frame, + 'x': x, + 'y': y, + 'width': width, + 'height': height + }); + } + blitImage(x, y, width, height, arr, offset, fromQueue) { if (this._renderQ.length !== 0 && !fromQueue) { // NB(directxman12): it's technically more performant here to use preallocated arrays, @@ -406,9 +417,16 @@ export default class Display { } } - drawImage(img, x, y) { - this._drawCtx.drawImage(img, x, y); - this._damage(x, y, img.width, img.height); + drawImage(img, ...args) { + this._drawCtx.drawImage(img, ...args); + + if (args.length <= 4) { + const [x, y] = args; + this._damage(x, y, img.width, img.height); + } else { + const [,, sw, sh, dx, dy] = args; + this._damage(dx, dy, sw, sh); + } } autoscale(containerWidth, containerHeight) { @@ -511,6 +529,35 @@ export default class Display { ready = false; } break; + case 'frame': + if (a.frame.ready) { + // The encoded frame may be larger than the rect due to + // limitations of the encoder, so we need to crop the + // frame. + let frame = a.frame.frame; + if (frame.codedWidth < a.width || frame.codedHeight < a.height) { + Log.Warn("Decoded video frame does not cover its full rectangle area. Expecting at least " + + a.width + "x" + a.height + " but got " + + frame.codedWidth + "x" + frame.codedHeight); + } + const sx = 0; + const sy = 0; + const sw = a.width; + const sh = a.height; + const dx = a.x; + const dy = a.y; + const dw = sw; + const dh = sh; + this.drawImage(frame, sx, sy, sw, sh, dx, dy, dw, dh); + frame.close(); + } else { + let display = this; + a.frame.promise.then(() => { + display._scanRenderQ(); + }); + ready = false; + } + break; } if (ready) { diff --git a/core/encodings.js b/core/encodings.js index 1a79989d..aa1fd4bb 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -15,6 +15,7 @@ export const encodings = { encodingZRLE: 16, encodingTightPNG: -260, encodingJPEG: 21, + encodingH264: 50, pseudoEncodingQualityLevel9: -23, pseudoEncodingQualityLevel0: -32, @@ -44,6 +45,7 @@ export function encodingName(num) { case encodings.encodingZRLE: return "ZRLE"; case encodings.encodingTightPNG: return "TightPNG"; case encodings.encodingJPEG: return "JPEG"; + case encodings.encodingH264: return "H.264"; default: return "[unknown encoding " + num + "]"; } } diff --git a/core/rfb.js b/core/rfb.js index f2deb0e7..9225cb46 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -10,7 +10,7 @@ import { toUnsigned32bit, toSigned32bit } from './util/int.js'; import * as Log from './util/logging.js'; import { encodeUTF8, decodeUTF8 } from './util/strings.js'; -import { dragThreshold } from './util/browser.js'; +import { dragThreshold, supportsWebCodecsH264Decode } from './util/browser.js'; import { clientToElement } from './util/element.js'; import { setCapture } from './util/events.js'; import EventTargetMixin from './util/eventtarget.js'; @@ -35,6 +35,7 @@ import TightDecoder from "./decoders/tight.js"; import TightPNGDecoder from "./decoders/tightpng.js"; import ZRLEDecoder from "./decoders/zrle.js"; import JPEGDecoder from "./decoders/jpeg.js"; +import H264Decoder from "./decoders/h264.js"; // How many seconds to wait for a disconnect to finish const DISCONNECT_TIMEOUT = 3; @@ -248,6 +249,7 @@ export default class RFB extends EventTargetMixin { this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); this._decoders[encodings.encodingZRLE] = new ZRLEDecoder(); this._decoders[encodings.encodingJPEG] = new JPEGDecoder(); + this._decoders[encodings.encodingH264] = new H264Decoder(); // NB: nothing that needs explicit teardown should be done // before this point, since this can throw an exception @@ -2115,6 +2117,9 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.encodingCopyRect); // Only supported with full depth support if (this._fbDepth == 24) { + if (supportsWebCodecsH264Decode) { + encs.push(encodings.encodingH264); + } encs.push(encodings.encodingTight); encs.push(encodings.encodingTightPNG); encs.push(encodings.encodingZRLE); diff --git a/core/util/browser.js b/core/util/browser.js index bbc9f5c1..1ecded66 100644 --- a/core/util/browser.js +++ b/core/util/browser.js @@ -70,6 +70,26 @@ try { } export const hasScrollbarGutter = _hasScrollbarGutter; +export let supportsWebCodecsH264Decode = false; + +async function _checkWebCodecsH264DecodeSupport() { + if (!('VideoDecoder' in window)) { + return; + } + + // We'll need to make do with some placeholders here + const config = { + codec: 'avc1.42401f', + codedWidth: 1920, + codedHeight: 1080, + optimizeForLatency: true, + }; + + const result = await VideoDecoder.isConfigSupported(config); + supportsWebCodecsH264Decode = result.supported; +} +_checkWebCodecsH264DecodeSupport(); + /* * The functions for detection of platforms and browsers below are exported * but the use of these should be minimized as much as possible. From bbb6a5b938d4dd85a99ca78718b09dc08bc9a81d Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 19 Aug 2024 14:01:00 +0200 Subject: [PATCH 09/13] Fix host and port via query string We need to call initSetting() even if we don't have any interesting default to set, as that is what checks if values have been provided as a query string. Fixes 96c76f7. --- app/ui.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/ui.js b/app/ui.js index 90dc7189..71933af6 100644 --- a/app/ui.js +++ b/app/ui.js @@ -159,6 +159,8 @@ const UI = { UI.updateLogging(); /* Populate the controls if defaults are provided in the URL */ + UI.initSetting('host', ''); + UI.initSetting('port', 0); UI.initSetting('encrypt', (window.location.protocol === "https:")); UI.initSetting('view_clip', false); UI.initSetting('resize', 'off'); From c1bba972f401b1e87e2266a05a647f44b3356020 Mon Sep 17 00:00:00 2001 From: Andri Yngvason Date: Sun, 11 Aug 2024 23:58:43 +0000 Subject: [PATCH 10/13] Add unit tests for H.264 decoder --- core/decoders/h264.js | 4 +- tests/test.h264.js | 264 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 tests/test.h264.js diff --git a/core/decoders/h264.js b/core/decoders/h264.js index db144fcd..2587d61b 100644 --- a/core/decoders/h264.js +++ b/core/decoders/h264.js @@ -9,7 +9,7 @@ import * as Log from '../util/logging.js'; -class H264Parser { +export class H264Parser { constructor(data) { this._data = data; this._index = 0; @@ -109,7 +109,7 @@ class H264Parser { } } -class H264Context { +export class H264Context { constructor(width, height) { this.lastUsed = 0; this._width = width; diff --git a/tests/test.h264.js b/tests/test.h264.js new file mode 100644 index 00000000..42273e7c --- /dev/null +++ b/tests/test.h264.js @@ -0,0 +1,264 @@ +import Websock from '../core/websock.js'; +import Display from '../core/display.js'; + +import { H264Parser } from '../core/decoders/h264.js'; +import H264Decoder from '../core/decoders/h264.js'; +import Base64 from '../core/base64.js'; + +import FakeWebSocket from './fake.websocket.js'; + +/* This is a 3 frame 16x16 video where the first frame is solid red, the second + * is solid green and the third is solid blue. + * + * The colour space is BT.709. It is encoded into the stream. + */ +const redGreenBlue16x16Video = new Uint8Array(Base64.decode( + 'AAAAAWdCwBTZnpuAgICgAAADACAAAAZB4oVNAAAAAWjJYyyAAAABBgX//4HcRem95tlIt5Ys' + + '2CDZI+7veDI2NCAtIGNvcmUgMTY0IHIzMTA4IDMxZTE5ZjkgLSBILjI2NC9NUEVHLTQgQVZD' + + 'IGNvZGVjIC0gQ29weWxlZnQgMjAwMy0yMDIzIC0gaHR0cDovL3d3dy52aWRlb2xhbi5vcmcv' + + 'eDI2NC5odG1sIC0gb3B0aW9uczogY2FiYWM9MCByZWY9NSBkZWJsb2NrPTE6MDowIGFuYWx5' + + 'c2U9MHgxOjB4MTExIG1lPWhleCBzdWJtZT04IHBzeT0xIHBzeV9yZD0xLjAwOjAuMDAgbWl4' + + 'ZWRfcmVmPTEgbWVfcmFuZ2U9MTYgY2hyb21hX21lPTEgdHJlbGxpcz0yIDh4OGRjdD0wIGNx' + + 'bT0wIGRlYWR6b25lPTIxLDExIGZhc3RfcHNraXA9MSBjaHJvbWFfcXBfb2Zmc2V0PS0yIHRo' + + 'cmVhZHM9MSBsb29rYWhlYWRfdGhyZWFkcz0xIHNsaWNlZF90aHJlYWRzPTAgbnI9MCBkZWNp' + + 'bWF0ZT0xIGludGVybGFjZWQ9MCBibHVyYXlfY29tcGF0PTAgY29uc3RyYWluZWRfaW50cmE9' + + 'MCBiZnJhbWVzPTAgd2VpZ2h0cD0wIGtleWludD1pbmZpbml0ZSBrZXlpbnRfbWluPTI1IHNj' + + 'ZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NTAgcmM9YWJyIG1idHJl' + + 'ZT0xIGJpdHJhdGU9NDAwIHJhdGV0b2w9MS4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02' + + 'OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAABZYiEBrxmKAAPVccAAS04' + + '4AA5DRJMnkycJk4TPwAAAAFBiIga8RigADVVHAAGaGOAANtuAAAAAUGIkBr///wRRQABVf8c' + + 'AAcho4AAiD4=')); + +let _haveH264Decode = null; + +async function haveH264Decode() { + if (_haveH264Decode !== null) { + return _haveH264Decode; + } + + if (!('VideoDecoder' in window)) { + _haveH264Decode = false; + return false; + } + + // We'll need to make do with some placeholders here + const config = { + codec: 'avc1.42401f', + codedWidth: 1920, + codedHeight: 1080, + optimizeForLatency: true, + }; + + _haveH264Decode = await VideoDecoder.isConfigSupported(config); + return _haveH264Decode; +} + +function createSolidColorFrameBuffer(color, width, height) { + const r = (color >> 24) & 0xff; + const g = (color >> 16) & 0xff; + const b = (color >> 8) & 0xff; + const a = (color >> 0) & 0xff; + + const size = width * height * 4; + let array = new Uint8ClampedArray(size); + + for (let i = 0; i < size / 4; ++i) { + array[i * 4 + 0] = r; + array[i * 4 + 1] = g; + array[i * 4 + 2] = b; + array[i * 4 + 3] = a; + } + + return array; +} + +function makeMessageHeader(length, resetContext, resetAllContexts) { + let flags = 0; + if (resetContext) { + flags |= 1; + } + if (resetAllContexts) { + flags |= 2; + } + + let header = new Uint8Array(8); + let i = 0; + + let appendU32 = (v) => { + header[i++] = (v >> 24) & 0xff; + header[i++] = (v >> 16) & 0xff; + header[i++] = (v >> 8) & 0xff; + header[i++] = v & 0xff; + }; + + appendU32(length); + appendU32(flags); + + return header; +} + +function wrapRectData(data, resetContext, resetAllContexts) { + let header = makeMessageHeader(data.length, resetContext, resetAllContexts); + return Array.from(header).concat(Array.from(data)); +} + +function testDecodeRect(decoder, x, y, width, height, data, display, depth) { + let sock; + let done = false; + + sock = new Websock; + sock.open("ws://example.com"); + + sock.on('message', () => { + done = decoder.decodeRect(x, y, width, height, sock, display, depth); + }); + + // Empty messages are filtered at multiple layers, so we need to + // do a direct call + if (data.length === 0) { + done = decoder.decodeRect(x, y, width, height, sock, display, depth); + } else { + sock._websocket._receiveData(new Uint8Array(data)); + } + + display.flip(); + + return done; +} + +function almost(a, b) { + let diff = Math.abs(a - b); + return diff < 5; +} + +describe('H.264 Parser', function () { + it('should parse constrained baseline video', function () { + let parser = new H264Parser(redGreenBlue16x16Video); + + let frame = parser.parse(); + expect(frame).to.have.property('key', true); + + expect(parser).to.have.property('profileIdc', 66); + expect(parser).to.have.property('constraintSet', 192); + expect(parser).to.have.property('levelIdc', 20); + + frame = parser.parse(); + expect(frame).to.have.property('key', false); + + frame = parser.parse(); + expect(frame).to.have.property('key', false); + + frame = parser.parse(); + expect(frame).to.be.null; + }); +}); + +describe('H.264 Decoder Unit Test', function () { + let decoder; + + beforeEach(async function () { + if (!await haveH264Decode()) { + this.skip(); + return; + } + decoder = new H264Decoder(); + }); + + it('creates and resets context', function () { + let context = decoder._getContext(1, 2, 3, 4); + expect(context._width).to.equal(3); + expect(context._height).to.equal(4); + expect(decoder._contexts).to.not.be.empty; + decoder._resetContext(1, 2, 3, 4); + expect(decoder._contexts).to.be.empty; + }); + + it('resets all contexts', function () { + decoder._getContext(0, 0, 1, 1); + decoder._getContext(2, 2, 1, 1); + expect(decoder._contexts).to.not.be.empty; + decoder._resetAllContexts(); + expect(decoder._contexts).to.be.empty; + }); + + it('caches contexts', function () { + let c1 = decoder._getContext(1, 2, 3, 4); + c1.lastUsed = 1; + let c2 = decoder._getContext(1, 2, 3, 4); + c2.lastUsed = 2; + expect(Object.keys(decoder._contexts).length).to.equal(1); + expect(c1.lastUsed).to.equal(c2.lastUsed); + }); + + it('deletes oldest context', function () { + for (let i = 0; i < 65; ++i) { + let context = decoder._getContext(i, 0, 1, 1); + context.lastUsed = i; + } + + expect(decoder._findOldestContextId()).to.equal('1,0,1,1'); + expect(decoder._contexts[decoder._contextId(0, 0, 1, 1)]).to.be.undefined; + expect(decoder._contexts[decoder._contextId(1, 0, 1, 1)]).to.not.be.null; + expect(decoder._contexts[decoder._contextId(63, 0, 1, 1)]).to.not.be.null; + expect(decoder._contexts[decoder._contextId(64, 0, 1, 1)]).to.not.be.null; + }); +}); + +describe('H.264 Decoder Functional Test', function () { + let decoder; + let display; + + before(FakeWebSocket.replace); + after(FakeWebSocket.restore); + + beforeEach(async function () { + if (!await haveH264Decode()) { + this.skip(); + return; + } + decoder = new H264Decoder(); + display = new Display(document.createElement('canvas')); + display.resize(16, 16); + }); + + it('should handle H.264 rect', async function () { + let data = wrapRectData(redGreenBlue16x16Video, false, false); + let done = testDecodeRect(decoder, 0, 0, 16, 16, data, display, 24); + expect(done).to.be.true; + await display.flush(); + let targetData = createSolidColorFrameBuffer(0x0000ffff, 16, 16); + expect(display).to.have.displayed(targetData, almost); + }); + + it('should handle specific context reset', async function () { + let data = wrapRectData(redGreenBlue16x16Video, false, false); + let done = testDecodeRect(decoder, 0, 0, 16, 16, data, display, 24); + expect(done).to.be.true; + await display.flush(); + let targetData = createSolidColorFrameBuffer(0x0000ffff, 16, 16); + expect(display).to.have.displayed(targetData, almost); + + data = wrapRectData([], true, false); + done = testDecodeRect(decoder, 0, 0, 16, 16, data, display, 24); + expect(done).to.be.true; + await display.flush(); + + expect(decoder._contexts[decoder._contextId(0, 0, 16, 16)]._decoder).to.be.null; + }); + + it('should handle global context reset', async function () { + let data = wrapRectData(redGreenBlue16x16Video, false, false); + let done = testDecodeRect(decoder, 0, 0, 16, 16, data, display, 24); + expect(done).to.be.true; + await display.flush(); + let targetData = createSolidColorFrameBuffer(0x0000ffff, 16, 16); + expect(display).to.have.displayed(targetData, almost); + + data = wrapRectData([], false, true); + done = testDecodeRect(decoder, 0, 0, 16, 16, data, display, 24); + expect(done).to.be.true; + await display.flush(); + + expect(decoder._contexts[decoder._contextId(0, 0, 16, 16)]._decoder).to.be.null; + }); +}); From a4465516df31f71136a25a91df98530489a4be81 Mon Sep 17 00:00:00 2001 From: Tomasz Kalisiak Date: Fri, 23 Aug 2024 13:14:36 +0200 Subject: [PATCH 11/13] Fix sQpushBytes sending the beginning of the array multiple times --- core/websock.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/websock.js b/core/websock.js index 21327c31..61a3091a 100644 --- a/core/websock.js +++ b/core/websock.js @@ -208,7 +208,7 @@ export default class Websock { chunkSize = bytes.length - offset; } - this._sQ.set(bytes.subarray(offset, chunkSize), this._sQlen); + this._sQ.set(bytes.subarray(offset, offset + chunkSize), this._sQlen); this._sQlen += chunkSize; offset += chunkSize; } From ffb4c0bf563a09903b0bca5ccc5483bed1522aba Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 29 Aug 2024 16:51:16 +0200 Subject: [PATCH 12/13] Let fake WebSocket handle large sends Dynamically grow the recorded send buffer if the test needs to send a lot of data. --- tests/fake.websocket.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/fake.websocket.js b/tests/fake.websocket.js index d273fe07..f1b30713 100644 --- a/tests/fake.websocket.js +++ b/tests/fake.websocket.js @@ -37,6 +37,15 @@ export default class FakeWebSocket { } else { data = new Uint8Array(data); } + if (this.bufferedAmount + data.length > this._sendQueue.length) { + let newlen = this._sendQueue.length; + while (this.bufferedAmount + data.length > newlen) { + newlen *= 2; + } + let newbuf = new Uint8Array(newlen); + newbuf.set(this._sendQueue); + this._sendQueue = newbuf; + } this._sendQueue.set(data, this.bufferedAmount); this.bufferedAmount += data.length; } From 50e4685bfff9c52a9de878bc095d4bfe7e4e39c2 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 29 Aug 2024 16:51:51 +0200 Subject: [PATCH 13/13] Fix tests for large WebSocket sends These failed to test that the data was correctly split as they only checked the first chunk transmitted. Use random values to avoid the risk of aligning our test data with the split boundaries and hence allowing false positives. --- tests/test.websock.js | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/tests/test.websock.js b/tests/test.websock.js index 53145b36..ab09f450 100644 --- a/tests/test.websock.js +++ b/tests/test.websock.js @@ -261,20 +261,15 @@ describe('Websock', function () { }); it('should implicitly split a large buffer', function () { let str = ''; - for (let i = 0;i <= bufferSize/5;i++) { - str += '\x12\x34\x56\x78\x90'; + let expected = []; + for (let i = 0;i < bufferSize * 3;i++) { + let byte = Math.random() * 0xff; + str += String.fromCharCode(byte); + expected.push(byte); } sock.sQpushString(str); - - let expected = []; - for (let i = 0;i < bufferSize/5;i++) { - expected.push(0x12); - expected.push(0x34); - expected.push(0x56); - expected.push(0x78); - expected.push(0x90); - } + sock.flush(); expect(sock).to.have.sent(new Uint8Array(expected)); }); @@ -308,24 +303,15 @@ describe('Websock', function () { }); it('should implicitly split a large buffer', function () { let buffer = []; - for (let i = 0;i <= bufferSize/5;i++) { - buffer.push(0x12); - buffer.push(0x34); - buffer.push(0x56); - buffer.push(0x78); - buffer.push(0x90); + let expected = []; + for (let i = 0;i < bufferSize * 3;i++) { + let byte = Math.random() * 0xff; + buffer.push(byte); + expected.push(byte); } sock.sQpushBytes(new Uint8Array(buffer)); - - let expected = []; - for (let i = 0;i < bufferSize/5;i++) { - expected.push(0x12); - expected.push(0x34); - expected.push(0x56); - expected.push(0x78); - expected.push(0x90); - } + sock.flush(); expect(sock).to.have.sent(new Uint8Array(expected)); });