From 419a3a70e956942d9cefef503d5586930ed7cae9 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Fri, 8 Sep 2023 13:12:28 -0400 Subject: [PATCH 01/41] WIP - multi monitor refactor --- README.md | 2 +- app/ui.js | 107 ++++--------- app/ui_screen.js | 40 +++++ core/display.js | 217 ++++++++++++++++++++++--- core/rfb.js | 371 ++++++++++++++++++++++++++++++------------- core/util/strings.js | 6 + screen.html | 95 +++++++++++ vnc.html | 31 ++-- 8 files changed, 640 insertions(+), 229 deletions(-) create mode 100644 app/ui_screen.js create mode 100644 screen.html diff --git a/README.md b/README.md index d77cb519..e2cb5564 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ For support with KasmVNC, post on the [KasmVNC Project](https://github.com/kasmt - Automatic mixing of webp and jpeg based on CPU availability on server - WebRTC UDP Transit - Lossless QOI Image format for Local LAN - - [Dynamic jpeg/webp image coompression](https://github.com/kasmtech/KasmVNC/wiki/Video-Rendering-Options#dynamic-image-quality) quality settings based on screen change rates + - [Dynamic jpeg/webp image compression](https://github.com/kasmtech/KasmVNC/wiki/Video-Rendering-Options#dynamic-image-quality) quality settings based on screen change rates - Seemless clipboard support (on Chromium based browsers) - Binary clipboard support for text, images, and formatted text (on Chromium based browsers) - Allow client to set/change most configuration settings diff --git a/app/ui.js b/app/ui.js index 3d674856..bc1ed064 100644 --- a/app/ui.js +++ b/app/ui.js @@ -31,8 +31,8 @@ window.updateSetting = (name, value) => { } } -import "core-js/stable"; -import "regenerator-runtime/runtime"; +//import "core-js/stable"; +//import "regenerator-runtime/runtime"; import * as Log from '../core/util/logging.js'; import _, { l10n } from './localization.js'; import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS, supportsPointerLock } @@ -69,6 +69,8 @@ const UI = { reconnectCallback: null, reconnectPassword: null, + supportsBroadcastChannel: (typeof BroadcastChannel !== "undefined"), + prime() { return WebUtil.initSettings().then(() => { if (document.readyState === "interactive" || document.readyState === "complete") { @@ -128,14 +130,13 @@ const UI = { UI.addExtraKeysHandlers(); UI.addGamingHandlers(); UI.addMachineHandlers(); - UI.addConnectionControlHandlers(); UI.addClipboardHandlers(); UI.addSettingsHandlers(); + UI.addMultiMonitorAddHandler(); document.getElementById("noVNC_status") .addEventListener('click', UI.hideStatus); UI.openControlbar(); - // UI.updateVisualState('init'); @@ -472,21 +473,6 @@ const UI = { .addEventListener('click', () => UI.rfb.machineReset()); }, - addConnectionControlHandlers() { - UI.addClickHandle('noVNC_disconnect_button', UI.disconnect); - - var connect_btn_el = document.getElementById("noVNC_connect_button"); - if (typeof(connect_btn_el) != 'undefined' && connect_btn_el != null) - { - connect_btn_el.addEventListener('click', UI.connect); - } - document.getElementById("noVNC_cancel_reconnect_button") - .addEventListener('click', UI.cancelReconnect); - - document.getElementById("noVNC_credentials_button") - .addEventListener('click', UI.setCredentials); - }, - addClipboardHandlers() { UI.addClickHandle('noVNC_clipboard_button', UI.toggleClipboardPanel); @@ -588,6 +574,13 @@ const UI = { window.addEventListener('msfullscreenchange', UI.updateFullscreenButton); }, + addMultiMonitorAddHandler() { + if (UI.supportsBroadcastChannel) { + UI.showControlInput("noVNC_addmonitor_button"); + UI.addClickHandle('noVNC_addmonitor_button', UI.addSecondaryMonitor); + } + }, + /* ------^------- * /EVENT HANDLERS * ============== @@ -676,8 +669,6 @@ const UI = { // State change closes dialogs as they may not be relevant // anymore UI.closeAllPanels(); - document.getElementById('noVNC_credentials_dlg') - .classList.remove('noVNC_open'); }, showStats() { @@ -1380,9 +1371,12 @@ const UI = { UI.rfb = new RFB(document.getElementById('noVNC_container'), document.getElementById('noVNC_keyboardinput'), url, - { shared: UI.getSetting('shared'), - repeaterID: UI.getSetting('repeaterID'), - credentials: { password: password } }); + { + shared: UI.getSetting('shared'), + repeaterID: UI.getSetting('repeaterID'), + credentials: { password: password } + }, + true ); UI.rfb.addEventListener("connect", UI.connectFinished); UI.rfb.addEventListener("disconnect", UI.disconnectFinished); UI.rfb.addEventListener("credentialsrequired", UI.credentials); @@ -1746,57 +1740,6 @@ const UI = { parent.postMessage({ action: 'clipboardrx', value: event.detail.text}, '*' ); //TODO fix star }, -/* ------^------- - * /CONNECTION - * ============== - * PASSWORD - * ------v------*/ - - credentials(e) { - // FIXME: handle more types - - document.getElementById("noVNC_username_block").classList.remove("noVNC_hidden"); - document.getElementById("noVNC_password_block").classList.remove("noVNC_hidden"); - - let inputFocus = "none"; - if (e.detail.types.indexOf("username") === -1) { - document.getElementById("noVNC_username_block").classList.add("noVNC_hidden"); - } else { - inputFocus = inputFocus === "none" ? "noVNC_username_input" : inputFocus; - } - if (e.detail.types.indexOf("password") === -1) { - document.getElementById("noVNC_password_block").classList.add("noVNC_hidden"); - } else { - inputFocus = inputFocus === "none" ? "noVNC_password_input" : inputFocus; - } - document.getElementById('noVNC_credentials_dlg') - .classList.add('noVNC_open'); - - setTimeout(() => document - .getElementById(inputFocus).focus(), 100); - - Log.Warn("Server asked for credentials"); - UI.showStatus(_("Credentials are required"), "warning"); - }, - - setCredentials(e) { - // Prevent actually submitting the form - e.preventDefault(); - - let inputElemUsername = document.getElementById('noVNC_username_input'); - const username = inputElemUsername.value; - - let inputElemPassword = document.getElementById('noVNC_password_input'); - const password = inputElemPassword.value; - // Clear the input after reading the password - inputElemPassword.value = ""; - - UI.rfb.sendCredentials({ username: username, password: password }); - UI.reconnectPassword = password; - document.getElementById('noVNC_credentials_dlg') - .classList.remove('noVNC_open'); - }, - /* ------^------- * /PASSWORD * ============== @@ -1867,6 +1810,20 @@ const UI = { UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi'); }, +/* ------^------- + * /MULTI-MONITOR SUPPORT + * ==============*/ + + addSecondaryMonitor() { + let new_display_path = window.location.pathname.replace(/[^/]*$/, '') + let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html`; + + Log.Debug(`Opening a secondary display ${new_display_url}`) + window.open(new_display_url); + }, + + + /* ------^------- * /RESIZE * ============== diff --git a/app/ui_screen.js b/app/ui_screen.js new file mode 100644 index 00000000..9ba4ff9a --- /dev/null +++ b/app/ui_screen.js @@ -0,0 +1,40 @@ + + + + +const UI = { + connected: false, + + //Initial Loading of the UI + prime() { + + }, + + //Render default UI + start() { + window.addEventListener("beforeunload", (e) => { + if (UI.rfb) { + UI.disconnect(); + } + }); + + + UI.addDefaultHandlers(); + }, + + addDefaultHandlers() { + document.getElementById('noVNC_connect_button', UI.connect); + }, + + connect() { + + }, + + disconnect() { + + } +} + +UI.prime(); + +export default UI; \ No newline at end of file diff --git a/core/display.js b/core/display.js index 1713314a..8b9c43a6 100644 --- a/core/display.js +++ b/core/display.js @@ -11,6 +11,7 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; +import { uuidv4 } from './util/strings.js' export default class Display { constructor(target) { @@ -56,7 +57,7 @@ export default class Display { this._targetCtx = this._target.getContext('2d'); // the visible canvas viewport (i.e. what actually gets seen) - this._viewportLoc = { 'x': 0, 'y': 0, 'w': this._target.width, 'h': this._target.height }; + //this._viewportLoc = { 'x': 0, 'y': 0, 'w': this._target.width, 'h': this._target.height }; Log.Debug("User Agent: " + navigator.userAgent); @@ -84,6 +85,21 @@ export default class Display { this._clipViewport = false; this._antiAliasing = 0; this._fps = 0; + this._screens = [{ + screenID: uuidv4(), + screenIndex: 0, + width: this._target.width, //client + height: this._target.height, //client + serverWidth: 0, //calculated + serverHeight: 0, //calculated + x: 0, + y: 0, + relativePosition: 0, + pixelRatio: window.devicePixelRatio, + containerHeight: this._target.parentNode.offsetHeight, + containerWidth: this._target.parentNode.offsetWidth, + channel: null + }]; // ===== EVENT HANDLERS ===== @@ -96,6 +112,8 @@ export default class Display { } // ===== PROPERTIES ===== + + get screens() { return this._screens; } get antiAliasing() { return this._antiAliasing; } set antiAliasing(value) { @@ -112,8 +130,8 @@ export default class Display { set clipViewport(viewport) { this._clipViewport = viewport; // May need to readjust the viewport dimensions - const vp = this._viewportLoc; - this.viewportChangeSize(vp.w, vp.h); + const vp = this._screens[0]; + this.viewportChangeSize(vp.width, vp.height); this.viewportChangePos(0, 0); } @@ -136,18 +154,167 @@ export default class Display { // ===== PUBLIC METHODS ===== + getScreenSize(resolutionQuality, max_width, max_height, hiDpi, disableLimit) { + let data = { + screens: null, + serverWidth: 0, + serverHeight: 0, + clientWidth: 0, + clientHeight: 0 + } + + //recalculate primary display container size + this._screens[0].containerHeight = this._target.parentNode.offsetHeight; + this._screens[0].containerWidth = this._target.parentNode.offsetWidth; + + //calculate server-side resolution of each screen + for (let i=0; i 1280 && !disableLimit && resolutionQuality == 1) { + height = 1280 * (height/width); //keeping the aspect ratio of original resolution, shrink y to match x + width = 1280; + } + //hard coded 720p + else if (resolutionQuality == 0 && !disableLimit) { + width = 1280; + height = 720; + } + //force full resolution on a high DPI monitor where the OS is scaling + else if (hiDpi) { + width = width * this._screens[i].pixelRatio; + height = height * this._screens[i].pixelRatio; + scale = 1 / this._screens[i].pixelRatio; + } + //physically small device with high DPI + else if (this._antiAliasing === 0 && this._screens[i].pixelRatio > 1 && width < 1000 & width > 0) { + Log.Info('Device Pixel ratio: ' + this._screens[i].pixelRatio + ' Reported Resolution: ' + width + 'x' + height); + let targetDevicePixelRatio = 1.5; + if (this._screens[i].pixelRatio > 2) { targetDevicePixelRatio = 2; } + let scaledWidth = (width * this._screens[i].pixelRatio) * (1 / targetDevicePixelRatio); + let scaleRatio = scaledWidth / x; + width = width * scaleRatio; + height = height * scaleRatio; + scale = 1 / scaleRatio; + Log.Info('Small device with hDPI screen detected, auto scaling at ' + scaleRatio + ' to ' + width + 'x' + height); + } + + this._screens[i].serverWidth = width; + this._screens[i].serverHeight = height; + + //this logic will only support monitors laid out side by side + //TODO: two vertically stacked monitors would require light refactoring here + data.serverWidth += width; + data.serverHeight = Math.max(data.serverHeight, height); + data.clientWidth += this._screens[i].width; + data.clientHeight = Math.max(data.clientHeight, this._screens[i].height); + this._screens[i].scale = scale; + } + + //calculate positions of monitors, this logic will only support two monitors side by side in either order + if (this._screens.length > 1) { + const primary_screen = this._screens[0]; + const secondary_screen = this._screens[1]; + switch (this._screens[1].relativePosition) { + case 0: + //primary screen is to left + total_width = secondary_screen.serverWidth + primary_screen.serverWidth; + total_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight); + secondary_screen.x = primary_screen.serverWidth; + if (secondary_screen.serverHeight < primary_screen.serverHeight) { + if ((total_height - secondary_screen.serverHeight) > 1) { + secondary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) + } + } else { + secondary_screen.y = 0; + if ((total_height - secondary_screen.serverHeight) > 1) { + this._screens[0].y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) + } + } + break; + case 2: + //primary screen is to right + total_width = primary_screen.serverWidth + secondary_screen.serverWidth; + total_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight); + this._screens[0].x = secondary_screen.serverWidth; + if (secondary_screen.serverHeight < primary_screen.serverHeight) { + if ((total_height - secondary_screen.serverHeight) > 1) { + secondary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) + } + } else { + secondary_screen.y = 0; + if ((total_height - secondary_screen.serverHeight) > 1) { + primary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) + } + } + break; + default: + //TODO: It would not be hard to support vertically stacked monitors + throw new Error("Unsupported screen orientation."); + } + } + + data.screens = structuredClone(this._screens); + + return data; + } + + addScreen(screenID, width, height, relativePosition, pixelRatio, containerHeight, containerWidth) { + //currently only support one secondary screen + if (this._screens.length > 1) { + this._screens[1].channel.close(); + this._screens.pop() + } + screenIdx = this.screens.length; + new_screen = { + screenID: screenID, + screenIndex: screenIdx, + width: width, //client + height: height, //client + serverWidth: 0, //calculated + serverHeight: 0, //calculated + x: 0, + y: 0, + relativePosition: relativePosition, + pixelRatio: pixelRatio, + containerHeight: containerHeight, + containerWidth: containerWidth, + channel: null, + scale: 0 + } + + new_screen.channel = new BroadcastChannel(`screen_${screenID}_channel`); + //new_screen.channel.message = this._handleSecondaryDisplayMessage().bind(this); + + this._screens.push(new_screen); + new_screen.channel.postMessage({ eventType: "registered", screenIndex: screenIdx }); + } + + removeScreen(screenID) { + for (let i=1; i= targetAspectRatio) { - scaleRatio = containerWidth / vp.w; + scaleRatio = containerWidth / vp.width; } else { - scaleRatio = containerHeight / vp.h; + scaleRatio = containerHeight / vp.height; } } @@ -658,14 +825,14 @@ export default class Display { _rescale(factor) { this._scale = factor; - const vp = this._viewportLoc; + const vp = this._screens[0]; // NB(directxman12): If you set the width directly, or set the // style width to a number, the canvas is cleared. // However, if you set the style width to a string // ('NNNpx'), the canvas is scaled without clearing. - const width = factor * vp.w + 'px'; - const height = factor * vp.h + 'px'; + const width = factor * vp.width + 'px'; + const height = factor * vp.height + 'px'; if ((this._target.style.width !== width) || (this._target.style.height !== height)) { @@ -673,12 +840,12 @@ export default class Display { this._target.style.height = height; } - Log.Info('Pixel Ratio: ' + window.devicePixelRatio + ', VNC Scale: ' + factor + 'VNC Res: ' + vp.w + 'x' + vp.h); + Log.Info('Pixel Ratio: ' + window.devicePixelRatio + ', VNC Scale: ' + factor + 'VNC Res: ' + vp.width + 'x' + vp.height); var pixR = Math.abs(Math.ceil(window.devicePixelRatio)); var isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; - if (this.antiAliasing === 2 || (this.antiAliasing === 0 && factor === 1 && this._target.style.imageRendering !== 'pixelated' && pixR === window.devicePixelRatio && vp.w > 0)) { + if (this.antiAliasing === 2 || (this.antiAliasing === 0 && factor === 1 && this._target.style.imageRendering !== 'pixelated' && pixR === window.devicePixelRatio && vp.width > 0)) { this._target.style.imageRendering = ((!isFirefox) ? 'pixelated' : 'crisp-edges' ); Log.Debug('Smoothing disabled'); } else if (this.antiAliasing === 1 || (this.antiAliasing === 0 && factor !== 1 && this._target.style.imageRendering !== 'auto')) { diff --git a/core/rfb.js b/core/rfb.js index b2ed01f5..bb6813a6 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -10,7 +10,7 @@ import { toUnsigned32bit, toSigned32bit } from './util/int.js'; import * as Log from './util/logging.js'; -import { encodeUTF8, decodeUTF8 } from './util/strings.js'; +import { encodeUTF8, decodeUTF8, uuidv4 } from './util/strings.js'; import { hashUInt8Array } from './util/int.js'; import { dragThreshold, supportsCursorURIs, isTouchDevice, isWindows, isMac, isIOS } from './util/browser.js'; import { clientToElement } from './util/element.js'; @@ -76,7 +76,7 @@ const extendedClipboardActionNotify = 1 << 27; const extendedClipboardActionProvide = 1 << 28; export default class RFB extends EventTargetMixin { - constructor(target, touchInput, urlOrChannel, options) { + constructor(target, touchInput, urlOrChannel, options, isPrimaryDisplay) { if (!target) { throw new Error("Must specify target"); } @@ -101,6 +101,7 @@ export default class RFB extends EventTargetMixin { this._shared = 'shared' in options ? !!options.shared : true; this._repeaterID = options.repeaterID || ''; this._wsProtocols = options.wsProtocols || ['binary']; + this._isPrimaryDisplay = (isPrimaryDisplay !== false); // Internal state this._rfbConnectionState = ''; @@ -122,7 +123,8 @@ export default class RFB extends EventTargetMixin { this._supportsContinuousUpdates = false; this._enabledContinuousUpdates = false; this._supportsSetDesktopSize = false; - this._screenID = 0; + this._screenID = uuidv4(); + this._screenIndex = 0; this._screenFlags = 0; this._qemuExtKeyEventSupported = false; @@ -212,6 +214,19 @@ export default class RFB extends EventTargetMixin { this._gestureLastMagnitudeX = 0; this._gestureLastMagnitudeY = 0; + // Secondary Displays + this._secondaryDisplays = {}; + this._supportsBroadcastChannel = (typeof BroadcastChannel !== "undefined"); + if (this._supportsBroadcastChannel) { + this._controlChannel = new BroadcastChannel("registrationChannel"); + this._controlChannel.message = this._handleControlMessage.bind(this); + Log.Debug("Attached to registrationChannel for secondary displays.") + + } + if (!this._isPrimaryDisplay) { + this._screenIndex = 2; + } + // Bound event handlers this._eventHandlers = { updateHiddenKeyboard: this._updateHiddenKeyboard.bind(this), @@ -283,68 +298,13 @@ export default class RFB extends EventTargetMixin { this._gestures = new GestureHandler(); - this._sock = new Websock(); - this._sock.on('message', () => { - this._handleMessage(); - }); - this._sock.on('open', () => { - if ((this._rfbConnectionState === 'connecting') && - (this._rfbInitState === '')) { - this._rfbInitState = 'ProtocolVersion'; - Log.Debug("Starting VNC handshake"); - } else { - this._fail("Unexpected server connection while " + - this._rfbConnectionState); - } - }); - this._sock.on('close', (e) => { - Log.Debug("WebSocket on-close event"); - let msg = ""; - if (e.code) { - msg = "(code: " + e.code; - if (e.reason) { - msg += ", reason: " + e.reason; - } - msg += ")"; - } - switch (this._rfbConnectionState) { - case 'connecting': - this._fail("Connection closed " + msg); - break; - case 'connected': - // Handle disconnects that were initiated server-side - this._updateConnectionState('disconnecting'); - this._updateConnectionState('disconnected'); - break; - case 'disconnecting': - // Normal disconnection path - this._updateConnectionState('disconnected'); - break; - case 'disconnected': - this._fail("Unexpected server disconnect " + - "when already disconnected " + msg); - break; - default: - this._fail("Unexpected server disconnect before connecting " + - msg); - break; - } - this._sock.off('close'); - // Delete reference to raw channel to allow cleanup. - this._rawChannel = null; - }); - this._sock.on('error', e => Log.Warn("WebSocket on-error event")); - - // Slight delay of the actual connection so that the caller has - // time to set up callbacks - setTimeout(this._updateConnectionState.bind(this, 'connecting')); + if (this._isPrimaryDisplay) { + this._setupWebSocket(); + } Log.Debug("<< RFB.constructor"); // ===== PROPERTIES ===== - - - this.dragViewport = false; this.focusOnClick = true; this.lastActiveAt = Date.now(); @@ -774,7 +734,9 @@ export default class RFB extends EventTargetMixin { if (this._rfbConnectionState === 'connected') { if (this._pendingApplyVideoRes) { - RFB.messages.setMaxVideoResolution(this._sock, this._maxVideoResolutionX, this._maxVideoResolutionY); + if (this._isPrimaryDisplay){ + RFB.messages.setMaxVideoResolution(this._sock, this._maxVideoResolutionX, this._maxVideoResolutionY); + } } if (this._pendingApplyResolutionChange) { @@ -793,15 +755,12 @@ export default class RFB extends EventTargetMixin { } disconnect() { - this._updateConnectionState('disconnecting'); - this._sock.off('error'); - this._sock.off('message'); - this._sock.off('open'); - } - - sendCredentials(creds) { - this._rfbCredentials = creds; - setTimeout(this._initMsg.bind(this), 0); + if (this._rfbConnectionState !== 'proxied') { + this._updateConnectionState('disconnecting'); + this._sock.off('error'); + this._sock.off('message'); + this._sock.off('open'); + } } sendCtrlAltDel() { @@ -851,13 +810,21 @@ export default class RFB extends EventTargetMixin { Log.Info("Sending key (" + (down ? "down" : "up") + "): keysym " + keysym + ", scancode " + scancode); - RFB.messages.QEMUExtendedKeyEvent(this._sock, keysym, down, scancode); + if (this._isPrimaryDisplay) { + RFB.messages.QEMUExtendedKeyEvent(this._sock, [ keysym, down, scancode]); + } else { + this._proxyRFBMessage('QEMUExtendedKeyEvent', [ keysym, down, scancode ]) + } } else { if (!keysym) { return; } Log.Info("Sending keysym (" + (down ? "down" : "up") + "): " + keysym); - RFB.messages.keyEvent(this._sock, keysym, down ? 1 : 0); + if (this._isPrimaryDisplay) { + RFB.messages.keyEvent(this._sock, keysym, down ? 1 : 0); + } else { + this._proxyRFBMessage('keyEvent', [ keysym, down ? 1 : 0 ]) + } } } @@ -909,7 +876,12 @@ export default class RFB extends EventTargetMixin { let mimes = [ 'text/plain' ]; dataset.push(data); - RFB.messages.sendBinaryClipboard(this._sock, dataset, mimes); + if (this._isPrimaryDisplay) { + RFB.messages.sendBinaryClipboard(this._sock, dataset, mimes); + } else { + this._proxyRFBMessage('sendBinaryClipboard', [ dataset, mimes ]); + } + } async clipboardPasteDataFrom(clipdata) { @@ -948,7 +920,7 @@ export default class RFB extends EventTargetMixin { continue; } - mimes.push(mime); + mimes.push(mime); dataset.push(data); Log.Debug('Sending mime type: ' + mime); break; @@ -973,23 +945,33 @@ export default class RFB extends EventTargetMixin { if (dataset.length > 0) { - RFB.messages.sendBinaryClipboard(this._sock, dataset, mimes); + if (this._isPrimaryDisplay) { + RFB.messages.sendBinaryClipboard(this._sock, dataset, mimes); + } else { + this._proxyRFBMessage('sendBinaryClipboard', [ dataset, mimes ]); + } } } requestBottleneckStats() { - RFB.messages.requestStats(this._sock); + if (this._isPrimaryDisplay) { + RFB.messages.requestStats(this._sock); + } } subscribeUnixRelay(name, processRelayFn) { - this._unixRelays = this._unixRelays || {}; - this._unixRelays[name] = processRelayFn; - RFB.messages.sendSubscribeUnixRelay(this._sock, name); + if (this._isPrimaryDisplay){ + this._unixRelays = this._unixRelays || {}; + this._unixRelays[name] = processRelayFn; + RFB.messages.sendSubscribeUnixRelay(this._sock, name); + } } sendUnixRelayData(name, payload) { - RFB.messages.sendUnixRelay(this._sock, name, payload); + if (this._isPrimaryDisplay) { + RFB.messages.sendUnixRelay(this._sock, name, payload); + } } // ===== PRIVATE METHODS ===== @@ -1003,10 +985,68 @@ export default class RFB extends EventTargetMixin { this._transitConnectionState = value; } + _setupWebSocket() { + this._sock = new Websock(); + this._sock.on('message', () => { + this._handleMessage(); + }); + this._sock.on('open', () => { + if ((this._rfbConnectionState === 'connecting') && + (this._rfbInitState === '')) { + this._rfbInitState = 'ProtocolVersion'; + Log.Debug("Starting VNC handshake"); + } else { + this._fail("Unexpected server connection while " + + this._rfbConnectionState); + } + }); + this._sock.on('close', (e) => { + Log.Debug("WebSocket on-close event"); + let msg = ""; + if (e.code) { + msg = "(code: " + e.code; + if (e.reason) { + msg += ", reason: " + e.reason; + } + msg += ")"; + } + switch (this._rfbConnectionState) { + case 'connecting': + this._fail("Connection closed " + msg); + break; + case 'connected': + // Handle disconnects that were initiated server-side + this._updateConnectionState('disconnecting'); + this._updateConnectionState('disconnected'); + break; + case 'disconnecting': + // Normal disconnection path + this._updateConnectionState('disconnected'); + break; + case 'disconnected': + this._fail("Unexpected server disconnect " + + "when already disconnected " + msg); + break; + default: + this._fail("Unexpected server disconnect before connecting " + + msg); + break; + } + this._sock.off('close'); + // Delete reference to raw channel to allow cleanup. + this._rawChannel = null; + }); + this._sock.on('error', e => Log.Warn("WebSocket on-error event")); + + // Slight delay of the actual connection so that the caller has + // time to set up callbacks + setTimeout(this._updateConnectionState.bind(this, 'connecting')); + } + _connect() { Log.Debug(">> RFB.connect"); - if (this._url) { + if (this._url && this._isPrimaryDisplay) { try { Log.Info(`connecting to ${this._url}`); this._sock.open(this._url, this._wsProtocols); @@ -1018,13 +1058,15 @@ export default class RFB extends EventTargetMixin { this._fail("Error when opening socket (" + e + ")"); } } - } else { + } else if (this._isPrimaryDisplay) { try { Log.Info(`attaching ${this._rawChannel} to Websock`); this._sock.attach(this._rawChannel); } catch (e) { this._fail("Error attaching channel (" + e + ")"); } + } else { + this._registerSecondaryDisplay(); } // Make our elements part of the page @@ -1085,7 +1127,7 @@ export default class RFB extends EventTargetMixin { this._resendClipboardNextUserDrivenEvent = true; // WebRTC UDP datachannel inits - if (typeof RTCPeerConnection !== 'undefined') { + if (typeof RTCPeerConnection !== 'undefined' && this._isPrimaryDisplay) { this._udpBuffer = new Map(); this._udpPeer = new RTCPeerConnection({ @@ -1190,7 +1232,7 @@ export default class RFB extends EventTargetMixin { } } - if (this._useUdp && typeof RTCPeerConnection !== 'undefined') { + if (this._useUdp && typeof RTCPeerConnection !== 'undefined' && this._isPrimaryDisplay) { setTimeout(function() { this._sendUdpUpgrade() }.bind(this), 3000); } @@ -1224,7 +1266,17 @@ export default class RFB extends EventTargetMixin { window.removeEventListener('focus', this._eventHandlers.handleFocusChange); this._keyboard.ungrab(); this._gestures.detach(); - this._sock.close(); + if (this._isPrimaryDisplay) { + this._sock.close(); + } else { + if (this._primaryDisplayChannel) { + this._primaryDisplayChannel.postMessage({eventType: 'unregister', screenID: this._screenID}) + this._primaryDisplayChannel.removeEventListener('message', this._handleSecondaryDisplayMessage); + this._primaryDisplayChannel.close(); + this._primaryDisplayChannel = null; + } + } + try { this._target.removeChild(this._screen); } catch (e) { @@ -1325,7 +1377,7 @@ export default class RFB extends EventTargetMixin { // When clipping is enabled, the screen is limited to // the size of the container. const size = this._screenSize(); - this._display.viewportChangeSize(size.w, size.h); + this._display.viewportChangeSize(size.screens[0].serverWidth, size.screens[0].serverHeight); this._fixScrollbars(); } } @@ -1335,7 +1387,7 @@ export default class RFB extends EventTargetMixin { this._display.scale = 1.0; } else { const size = this._screenSize(false); - this._display.autoscale(size.w, size.h, size.scale); + this._display.autoscale(size.screens[0].containerWidth, size.screens[0].containerHeight, size.screens[0].scale); } this._fixScrollbars(); } @@ -1351,20 +1403,20 @@ export default class RFB extends EventTargetMixin { return; } const size = this._screenSize(); - RFB.messages.setDesktopSize(this._sock, - Math.floor(size.w), Math.floor(size.h), - this._screenID, this._screenFlags); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); Log.Debug('Requested new desktop size: ' + - size.w + 'x' + size.h); + size.serverWidth + 'x' + size.serverHeight); } // Gets the the size of the available screen _screenSize (limited) { + return this._display.getScreenSize(this.videoQuality, this.forcedResolutionX, this.forcedResolutionY, this._hiDpi, limited); + if (limited === undefined) { limited = true; } - var x = this.forcedResolutionX || this._screen.offsetWidth; + var x = this.forcedResolutionX || this._screen.offsetWidth; var y = this.forcedResolutionY || this._screen.offsetHeight; var scale = 0; // 0=auto try { @@ -1469,6 +1521,10 @@ export default class RFB extends EventTargetMixin { } break; + case 'proxied': + //secondary display that needs to proxy messages through the broadcast channel + break; + default: Log.Error("Unknown connection state: " + state); return; @@ -1486,7 +1542,9 @@ export default class RFB extends EventTargetMixin { this._disconnTimer = null; // make sure we don't get a double event - this._sock.off('close'); + if (this._rfbConnectionState !== 'proxied') { + this._sock.off('close'); + } } switch (state) { @@ -1550,6 +1608,62 @@ export default class RFB extends EventTargetMixin { { detail: { capabilities: this._capabilities } })); } + _registerSecondaryDisplay() { + this._primaryDisplayChannel = new BroadcastChannel(`screen_${this._screenID}_channel`); + this._primaryDisplayChannel.message = this._handleSecondaryDisplayMessage.bind(this); + const size = this._screenSize(); + + message = { + eventType: 'register', + screenID: this._screenID, + screenIndex: this._screenIndex, + width: size.w, + height: size.h, + x: 0, + y: 0, + relativePosition: 0, + pixelRatio: window.devicePixelRatio, + containerWidth: this._screen.offsetWidth, + containerHeight: this._screen.offsetWidth, + channel: null + } + this._controlChannel.postMessage(message) + } + + _proxyRFBMessage(messageType, data) { + message = { + messageType: messageType, + data: data + } + this._controlChannel.postMessage(message); + } + + _handleControlMessage(event) { + console.log(event); + + switch (event.eventType) { + case 'register': + this._display.addScreen(event.screenID, event.width, event.height, event.relativePosition, event.pixelRatio, event.containerHeight, event.containerWidth); + Log.Info(`Secondary monitor (${event.screenID}) has been registered.`); + break; + case 'unregister': + if (this._display.removeScreen(event.screenID)) { + Log.Info(`Secondary monitor (${event.screenID}) has been removed.`); + } else { + Log.Info(`Secondary monitor (${event.screenID}) not found.`); + } + } + + + } + + _handleSecondaryDisplayMessage(event) { + console.log("Message Received: " + event); + if (this._isPrimaryDisplay) { + + } + } + _handleMessage() { if (this._sock.rQlen === 0) { Log.Warn("handleMessage called on an empty receive queue"); @@ -1790,16 +1904,23 @@ export default class RFB extends EventTargetMixin { //console.log("new_pos x" + x + ", y" + y); //console.log("lock x " + this._pointerLockPos.x + ", y " + this._pointerLockPos.y); //console.log("rel x " + rel_16_x + ", y " + rel_16_y); - - RFB.messages.pointerEvent(this._sock, rel_16_x, - rel_16_y, mask); + if (this._isPrimaryDisplay){ + RFB.messages.pointerEvent(this._sock, rel_16_x, rel_16_y, mask); + } else { + this._proxyRFBMessage('pointerEvent', [ rel_16_x, rel_16_y, mask ]); + } + // reset the cursor position to center this._mousePos = { x: this._pointerLockPos.x , y: this._pointerLockPos.y }; this._cursor.move(this._pointerLockPos.x, this._pointerLockPos.y); } else { - RFB.messages.pointerEvent(this._sock, this._display.absX(x), - this._display.absY(y), mask); + if (this._isPrimaryDisplay) { + RFB.messages.pointerEvent(this._sock, this._display.absX(x), this._display.absY(y), mask); + } else { + this._proxyRFBMessage('pointerEvent', [ this._display.absX(x), this._display.absY(y), mask ]); + } + } } @@ -1808,7 +1929,12 @@ export default class RFB extends EventTargetMixin { if (this._rfbConnectionState !== 'connected') { return; } if (this._viewOnly) { return; } // View only, skip mouse events - RFB.messages.pointerEvent(this._sock, this._display.absX(x), this._display.absY(y), 0, dX, dY); + if (this.isPrimaryDisplay){ + RFB.messages.pointerEvent(this._sock, this._display.absX(x), this._display.absY(y), 0, dX, dY); + } else { + this._proxyRFBMessage('pointerEvent', [ this._display.absX(x), this._display.absY(y), 0, dX, dY ]); + } + } _handleWheel(ev) { @@ -3570,7 +3696,7 @@ export default class RFB extends EventTargetMixin { for (let i = 0; i < numberOfScreens; i += 1) { // Save the id and flags of the first screen if (i === 0) { - this._screenID = this._sock.rQshiftBytes(4); // id + this._screenIndex = this._sock.rQshiftBytes(4); // id this._sock.rQskipBytes(2); // x-position this._sock.rQskipBytes(2); // y-position this._sock.rQskipBytes(2); // width @@ -4061,20 +4187,50 @@ RFB.messages = { } }, - setDesktopSize(sock, width, height, id, flags) { + setDesktopSize(sock, size, flags) { const buff = sock._sQ; const offset = sock._sQlen; buff[offset] = 251; // msg-type buff[offset + 1] = 0; // padding - buff[offset + 2] = width >> 8; // width - buff[offset + 3] = width; - buff[offset + 4] = height >> 8; // height - buff[offset + 5] = height; + buff[offset + 2] = size.serverWidth >> 8; // width + buff[offset + 3] = size.serverWidth; + buff[offset + 4] = size.serverHeight >> 8; // height + buff[offset + 5] = size.serverHeight; - buff[offset + 6] = 1; // number-of-screens + buff[offset + 6] = size.screens.length; // number-of-screens buff[offset + 7] = 0; // padding + let i = 8; + for (let iS = 0; iS < size.screens.length; iS++) { + //screen id + buff[offset + i++] = iS >> 24; + buff[offset + i++] = iS >> 16; + buff[offset + i++] = iS >> 8; + buff[offset + i++] = iS; + //screen x position + buff[offset + i++] = size.screens[iS].x >> 8; + buff[offset + i++] = size.screens[iS].x; + //screen y position + buff[offset + i++] = size.screens[iS].y >> 8; + buff[offset + i++] = size.screens[iS].y; + //screen width + buff[offset + i++] = size.screens[iS].serverWidth >> 8; + buff[offset + i++] = size.screens[iS].serverWidth; + //screen height + buff[offset + i++] = size.screens[iS].serverHeight >> 8; + buff[offset + i++] = size.screens[iS].serverHeight; + //flags + buff[offset + i++] = flags >> 24; + buff[offset + i++] = flags >> 16; + buff[offset + i++] = flags >> 8; + buff[offset + i++] = flags; + } + + sock._sQlen += i; + sock.flush(); + + /* // screen array buff[offset + 8] = id >> 24; // id buff[offset + 9] = id >> 16; @@ -4095,6 +4251,7 @@ RFB.messages = { sock._sQlen += 24; sock.flush(); + */ }, setMaxVideoResolution(sock, width, height) { diff --git a/core/util/strings.js b/core/util/strings.js index 3dd4b29f..4f6241b0 100644 --- a/core/util/strings.js +++ b/core/util/strings.js @@ -26,3 +26,9 @@ export function decodeUTF8(utf8string, allowLatin1=false) { export function encodeUTF8(DOMString) { return unescape(encodeURIComponent(DOMString)); } + +export function uuidv4() { + return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c => + (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16) + ); + } diff --git a/screen.html b/screen.html new file mode 100644 index 00000000..e0d3fb79 --- /dev/null +++ b/screen.html @@ -0,0 +1,95 @@ + + + + + + KasmVNC + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
KasmVNC encountered an error:
+
+
+
+
+ + + +
+ + +
+
+
+
+ Connect +
+
+
+
+ + +
+ + +
+ \ No newline at end of file diff --git a/vnc.html b/vnc.html index d334d983..08d581fc 100644 --- a/vnc.html +++ b/vnc.html @@ -50,7 +50,7 @@ - + @@ -185,6 +185,14 @@ Fullscreen + +
+ + Add Monitor +
+
- -
-
-
    -
  • - - -
    • -
  • - - -
    • -
  • - -
    • -
-
-
-
From 96001175a6a1f49550adfef7d0b5ed185e7d71a8 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Tue, 12 Sep 2023 14:16:19 -0400 Subject: [PATCH 02/41] WIP - computer acting odd, committing code --- app/ui_screen.js | 206 +++++++++++++++++++++++++- core/display.js | 322 ++++++++++++++++++++++++++++++----------- core/input/keyboard.js | 7 +- core/rfb.js | 94 +++++++----- 4 files changed, 497 insertions(+), 132 deletions(-) diff --git a/app/ui_screen.js b/app/ui_screen.js index 9ba4ff9a..b772e39f 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -1,13 +1,15 @@ - - - +import RFB from "../core/rfb.js"; +import * as WebUtil from "./webutil.js"; +import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS, supportsPointerLock } + from '../core/util/browser.js'; +import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js"; const UI = { connected: false, //Initial Loading of the UI prime() { - + this.start(); }, //Render default UI @@ -20,19 +22,209 @@ const UI = { UI.addDefaultHandlers(); + UI.updateVisualState('disconnected'); }, addDefaultHandlers() { - document.getElementById('noVNC_connect_button', UI.connect); + document.getElementById('noVNC_connect_button').addEventListener('click', UI.connect);; + }, + + getSetting(name, isBool) { + const ctrl = document.getElementById('noVNC_setting_' + name); + let val = WebUtil.readSetting(name); + if (typeof val !== 'undefined' && val !== null && isBool) { + if (val.toString().toLowerCase() in {'0': 1, 'no': 1, 'false': 1}) { + val = false; + } else { + val = true; + } + } + return val; }, connect() { - + UI.rfb = new RFB(document.getElementById('noVNC_container'), + document.getElementById('noVNC_keyboardinput'), + "", //URL + { + shared: UI.getSetting('shared', true), + repeaterID: UI.getSetting('repeaterID', false), + credentials: { password: null } + }, + false // Not a primary display + ); + UI.rfb.addEventListener("connect", UI.connectFinished); + //UI.rfb.addEventListener("disconnect", UI.disconnectFinished); + UI.rfb.clipViewport = UI.getSetting('view_clip'); + UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; + UI.rfb.resizeSession = UI.getSetting('resize') === 'remote'; + UI.rfb.qualityLevel = parseInt(UI.getSetting('quality')); + UI.rfb.dynamicQualityMin = parseInt(UI.getSetting('dynamic_quality_min')); + UI.rfb.dynamicQualityMax = parseInt(UI.getSetting('dynamic_quality_max')); + UI.rfb.jpegVideoQuality = parseInt(UI.getSetting('jpeg_video_quality')); + UI.rfb.webpVideoQuality = parseInt(UI.getSetting('webp_video_quality')); + UI.rfb.videoArea = parseInt(UI.getSetting('video_area')); + UI.rfb.videoTime = parseInt(UI.getSetting('video_time')); + UI.rfb.videoOutTime = parseInt(UI.getSetting('video_out_time')); + UI.rfb.videoScaling = parseInt(UI.getSetting('video_scaling')); + UI.rfb.treatLossless = parseInt(UI.getSetting('treat_lossless')); + UI.rfb.maxVideoResolutionX = parseInt(UI.getSetting('max_video_resolution_x')); + UI.rfb.maxVideoResolutionY = parseInt(UI.getSetting('max_video_resolution_y')); + UI.rfb.frameRate = parseInt(UI.getSetting('framerate')); + UI.rfb.compressionLevel = parseInt(UI.getSetting('compression')); + UI.rfb.showDotCursor = UI.getSetting('show_dot'); + UI.rfb.idleDisconnect = UI.getSetting('idle_disconnect'); + UI.rfb.pointerRelative = UI.getSetting('pointer_relative'); + UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality')); + UI.rfb.antiAliasing = UI.getSetting('anti_aliasing'); + UI.rfb.clipboardUp = UI.getSetting('clipboard_up'); + UI.rfb.clipboardDown = UI.getSetting('clipboard_down'); + UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless'); + UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime'); + UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless; + UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc'); + UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi'); + UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper(); + if (UI.rfb.videoQuality === 5) { + UI.rfb.enableQOI = true; + } + }, + + updateVisualState(state) { + document.documentElement.classList.remove("noVNC_connecting"); + document.documentElement.classList.remove("noVNC_connected"); + document.documentElement.classList.remove("noVNC_disconnecting"); + document.documentElement.classList.remove("noVNC_reconnecting"); + document.documentElement.classList.remove("noVNC_disconnected"); + + const transitionElem = document.getElementById("noVNC_transition_text"); + if (WebUtil.isInsideKasmVDI()) + { + parent.postMessage({ action: 'connection_state', value: state}, '*' ); + } + + switch (state) { + case 'init': + break; + case 'connecting': + transitionElem.textContent = _("Connecting..."); + document.documentElement.classList.add("noVNC_connecting"); + break; + case 'connected': + document.documentElement.classList.add("noVNC_connected"); + break; + case 'disconnecting': + transitionElem.textContent = _("Disconnecting..."); + document.documentElement.classList.add("noVNC_disconnecting"); + break; + case 'disconnected': + document.documentElement.classList.add("noVNC_disconnected"); + break; + case 'reconnecting': + transitionElem.textContent = _("Reconnecting..."); + document.documentElement.classList.add("noVNC_reconnecting"); + break; + default: + Log.Error("Invalid visual state: " + state); + UI.showStatus(_("Internal error"), 'error'); + return; + } + }, + + showStatus(text, statusType, time, kasm = false) { + // If inside the full Kasm CDI framework, don't show messages unless explicitly told to + if (WebUtil.isInsideKasmVDI() && !kasm) { + return; + } + + const statusElem = document.getElementById('noVNC_status'); + + if (typeof statusType === 'undefined') { + statusType = 'normal'; + } + + // Don't overwrite more severe visible statuses and never + // errors. Only shows the first error. + if (statusElem.classList.contains("noVNC_open")) { + if (statusElem.classList.contains("noVNC_status_error")) { + return; + } + if (statusElem.classList.contains("noVNC_status_warn") && + statusType === 'normal') { + return; + } + } + + clearTimeout(UI.statusTimeout); + + switch (statusType) { + case 'error': + statusElem.classList.remove("noVNC_status_warn"); + statusElem.classList.remove("noVNC_status_normal"); + statusElem.classList.add("noVNC_status_error"); + break; + case 'warning': + case 'warn': + statusElem.classList.remove("noVNC_status_error"); + statusElem.classList.remove("noVNC_status_normal"); + statusElem.classList.add("noVNC_status_warn"); + break; + case 'normal': + case 'info': + default: + statusElem.classList.remove("noVNC_status_error"); + statusElem.classList.remove("noVNC_status_warn"); + statusElem.classList.add("noVNC_status_normal"); + break; + } + + statusElem.textContent = text; + statusElem.classList.add("noVNC_open"); + + // If no time was specified, show the status for 1.5 seconds + if (typeof time === 'undefined') { + time = 1500; + } + + // Error messages do not timeout + if (statusType !== 'error') { + UI.statusTimeout = window.setTimeout(UI.hideStatus, time); + } }, disconnect() { - } + }, + + connectFinished(e) { + UI.connected = true; + UI.inhibitReconnect = false; + + UI.showStatus("Secondary Screen Connected"); + UI.updateVisualState('connected'); + + // Do this last because it can only be used on rendered elements + UI.rfb.focus(); + }, + + initMouseButtonMapper() { + const mouseButtonMapper = new MouseButtonMapper(); + + const settings = WebUtil.readSetting("mouseButtonMapper"); + if (settings) { + mouseButtonMapper.load(settings); + return mouseButtonMapper; + } + + mouseButtonMapper.set(0, XVNC_BUTTONS.LEFT_BUTTON); + mouseButtonMapper.set(1, XVNC_BUTTONS.MIDDLE_BUTTON); + mouseButtonMapper.set(2, XVNC_BUTTONS.RIGHT_BUTTON); + mouseButtonMapper.set(3, XVNC_BUTTONS.BACK_BUTTON); + mouseButtonMapper.set(4, XVNC_BUTTONS.FORWARD_BUTTON); + WebUtil.writeSetting("mouseButtonMapper", mouseButtonMapper.dump()); + + return mouseButtonMapper; + }, } UI.prime(); diff --git a/core/display.js b/core/display.js index 8b9c43a6..fa003a66 100644 --- a/core/display.js +++ b/core/display.js @@ -14,7 +14,7 @@ import { isWindows } from './util/browser.js'; import { uuidv4 } from './util/strings.js' export default class Display { - constructor(target) { + constructor(target, isPrimaryDisplay) { Log.Debug(">> Display.constructor"); /* @@ -85,8 +85,10 @@ export default class Display { this._clipViewport = false; this._antiAliasing = 0; this._fps = 0; + this._isPrimaryDisplay = isPrimaryDisplay; + this._screenID = uuidv4(); this._screens = [{ - screenID: uuidv4(), + screenID: this._screenID, screenIndex: 0, width: this._target.width, //client height: this._target.height, //client @@ -108,6 +110,11 @@ export default class Display { // Use requestAnimationFrame to write to canvas, to match display refresh rate this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + if (!this._isPrimaryDisplay) { + this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`); + this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this)); + } + Log.Debug("<< Display.constructor"); } @@ -166,6 +173,8 @@ export default class Display { //recalculate primary display container size this._screens[0].containerHeight = this._target.parentNode.offsetHeight; this._screens[0].containerWidth = this._target.parentNode.offsetWidth; + this._screens[0].width = this._target.width; + this._screens[0].height = this._target.height; //calculate server-side resolution of each screen for (let i=0; i 1) { const primary_screen = this._screens[0]; const secondary_screen = this._screens[1]; + let total_width = 0; + let total_height = 0; + switch (this._screens[1].relativePosition) { case 0: //primary screen is to left @@ -257,50 +269,59 @@ export default class Display { } } - data.screens = structuredClone(this._screens); + data.screens = this._screens; return data; } addScreen(screenID, width, height, relativePosition, pixelRatio, containerHeight, containerWidth) { - //currently only support one secondary screen - if (this._screens.length > 1) { - this._screens[1].channel.close(); - this._screens.pop() - } - screenIdx = this.screens.length; - new_screen = { - screenID: screenID, - screenIndex: screenIdx, - width: width, //client - height: height, //client - serverWidth: 0, //calculated - serverHeight: 0, //calculated - x: 0, - y: 0, - relativePosition: relativePosition, - pixelRatio: pixelRatio, - containerHeight: containerHeight, - containerWidth: containerWidth, - channel: null, - scale: 0 - } + if (this._isPrimaryDisplay) { + //currently only support one secondary screen + if (this._screens.length > 1) { + this._screens[1].channel.close(); + this._screens.pop() + } - new_screen.channel = new BroadcastChannel(`screen_${screenID}_channel`); - //new_screen.channel.message = this._handleSecondaryDisplayMessage().bind(this); + var new_screen = { + screenID: screenID, + screenIndex: this.screens.length, + width: width, //client + height: height, //client + serverWidth: 0, //calculated + serverHeight: 0, //calculated + x: 0, + y: 0, + relativePosition: relativePosition, + pixelRatio: pixelRatio, + containerHeight: containerHeight, + containerWidth: containerWidth, + channel: null, + scale: 0 + } - this._screens.push(new_screen); - new_screen.channel.postMessage({ eventType: "registered", screenIndex: screenIdx }); + new_screen.channel = new BroadcastChannel(`screen_${screenID}_channel`); + //new_screen.channel.message = this._handleSecondaryDisplayMessage().bind(this); + + this._screens.push(new_screen); + new_screen.channel.postMessage({ eventType: "registered", screenIndex: new_screen.screenIndex }); + } else { + throw new Error("Cannot add a screen to a secondary display.") + } + } removeScreen(screenID) { - for (let i=1; i 0) { + width = this._screens[0].serverWidth; + height = this._screens[0].serverHeight; + } if (canvas.width !== width || canvas.height !== height) { // We have to save the canvas data since changing the size will clear it @@ -434,7 +459,8 @@ export default class Display { this._asyncRenderQPush({ 'type': 'flip', 'frame_id': frame_id, - 'rect_cnt': rect_cnt + 'rect_cnt': rect_cnt, + 'screenLocations': [ { screenIndex: 0, x: 0, y: 0 } ] }); } @@ -480,15 +506,17 @@ export default class Display { fillRect(x, y, width, height, color, frame_id, fromQueue) { if (!fromQueue) { - this._asyncRenderQPush({ - 'type': 'fill', - 'x': x, - 'y': y, - 'width': width, - 'height': height, - 'color': color, - 'frame_id': frame_id - }); + let rect = { + type: 'fill', + x: x, + y: y, + width: width, + height: height, + color: color, + frame_id: frame_id + } + this._processRectScreens(rect); + this._asyncRenderQPush(rect); } else { this._setFillColor(color); this._targetCtx.fillRect(x, y, width, height); @@ -497,7 +525,7 @@ export default class Display { copyImage(oldX, oldY, newX, newY, w, h, frame_id, fromQueue) { if (!fromQueue) { - this._asyncRenderQPush({ + let rect = { 'type': 'copy', 'oldX': oldX, 'oldY': oldY, @@ -506,7 +534,9 @@ export default class Display { 'width': w, 'height': h, 'frame_id': frame_id - }); + } + this._processRectScreens(rect); + this._asyncRenderQPush(rect); } else { // Due to this bug among others [1] we need to disable the image-smoothing to // avoid getting a blur effect when copying data. @@ -531,18 +561,31 @@ export default class Display { if ((width === 0) || (height === 0)) { return; } - const img = new Image(); - img.src = "data: " + mime + ";base64," + Base64.encode(arr); - - this._asyncRenderQPush({ + + let rect = { 'type': 'img', - 'img': img, + 'img': null, 'x': x, 'y': y, 'width': width, 'height': height, 'frame_id': frame_id - }); + } + this._processRectScreens(rect); + + if (rect.inPrimary) { + const img = new Image(); + img.src = "data: " + mime + ";base64," + Base64.encode(arr); + rect.img = img; + } else { + rect.type = 'img_array' + } + if (rect.inSecondary) { + rect.mime = mime; + rect.arr = arr; + } + + this._asyncRenderQPush(rect); } transparentRect(x, y, width, height, img, frame_id) { @@ -560,12 +603,18 @@ export default class Display { 'height': height, 'frame_id': frame_id } + this._processRectScreens(rect); - let imageBmpPromise = createImageBitmap(img); - imageBmpPromise.then( function(img) { - rect.img = img; - rect.img.complete = true; - }.bind(rect) ); + if (rect.inPrimary) { + let imageBmpPromise = createImageBitmap(img); + imageBmpPromise.then( function(img) { + rect.img = img; + rect.img.complete = true; + }.bind(rect) ); + } + if (rect.inSecondary) { + rect.arr = img; + } this._asyncRenderQPush(rect); } @@ -577,7 +626,7 @@ export default class Display { // this probably isn't getting called *nearly* as much const newArr = new Uint8Array(width * height * 4); newArr.set(new Uint8Array(arr.buffer, 0, newArr.length)); - this._asyncRenderQPush({ + let rect = { 'type': 'blit', 'data': newArr, 'x': x, @@ -585,7 +634,9 @@ export default class Display { 'width': width, 'height': height, 'frame_id': frame_id - }); + } + this._processRectScreens(rect); + this._asyncRenderQPush(rect); } else { // NB(directxman12): arr must be an Type Array view let data = new Uint8ClampedArray(arr.buffer, @@ -598,7 +649,7 @@ export default class Display { blitQoi(x, y, width, height, arr, offset, frame_id, fromQueue) { if (!fromQueue) { - this._asyncRenderQPush({ + let rect = { 'type': 'blitQ', 'data': arr, 'x': x, @@ -606,7 +657,9 @@ export default class Display { 'width': width, 'height': height, 'frame_id': frame_id - }); + } + this._processRectScreens(rect); + this._asyncRenderQPush(rect); } else { this._targetCtx.putImageData(arr, x, y); } @@ -647,6 +700,53 @@ export default class Display { // ===== PRIVATE METHODS ===== + _handleSecondaryDisplayMessage(event) { + if (!this._isPrimaryDisplay && event.data) { + + switch (event.data.eventType) { + case 'rect': + let rect = event.data.rect; + let pos = rect.screenLocations[event.data.screenLocationIndex]; + if (!pos) { + console.log('wtf'); + } + switch (rect.type) { + case 'copy': + this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); + break; + case 'fill': + this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); + break; + case 'blit': + this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + break; + case 'blitQ': + this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + break; + case 'img': + case 'img_arr': + rect.img = new Image(); + rect.img.src = "data: " + rect.mime + ";base64," + Base64.encode(rect.arr); + if (!rect.img.complete) { + rect.img.addEventListener('load', function (rect) { + this.drawImage(rect.img, pos.x, pos.y, rect.width, rect.height); + }.bind(this, rect)); + } else { + this.drawImage(rect.img, pos.x, pos.y, rect.width, rect.height); + } + break; + case 'transparent': + let imageBmpPromise = createImageBitmap(rect.arr); + imageBmpPromise.then(function(rect, img) { + this.drawImage(img, pos.x, pos.y, rect.width, rect.height); + }).bind(this, rect); + break; + } + break; + } + } + } + /* Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding */ @@ -772,34 +872,53 @@ export default class Display { for (let i = 0; i < frame.length; i++) { const a = frame[i]; - switch (a.type) { - case 'copy': - this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, a.frame_id, true); - break; - case 'fill': - this.fillRect(a.x, a.y, a.width, a.height, a.color, a.frame_id, true); - break; - case 'blit': - this.blitImage(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true); - break; - case 'blitQ': - this.blitQoi(a.x, a.y, a.width, a.height, a.data, 0, a.frame_id, true); - break; - case 'img': - this.drawImage(a.img, a.x, a.y, a.width, a.height); - break; - case 'transparent': - transparent_rects.push(a); - break; + + for (let sI = 0; sI < a.screenLocations.length; sI++) { + let screenLocation = a.screenLocations[sI]; + if (screenLocation.screenIndex == 0) { + switch (a.type) { + case 'copy': + this.copyImage(screenLocation.oldX, screenLocation.oldY, screenLocation.x, screenLocation.y, a.width, a.height, a.frame_id, true); + break; + case 'fill': + this.fillRect(screenLocation.x, screenLocation.y, a.width, a.height, a.color, a.frame_id, true); + break; + case 'blit': + this.blitImage(screenLocation.x, screenLocation.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'blitQ': + this.blitQoi(screenLocation.x, screenLocation.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'img': + this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height); + break; + case 'transparent': + transparent_rects.push(a); + break; + } + } else { + if (a.img) { + a.img = null; + } + this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + } } } //rects with transparency get applied last for (let i = 0; i < transparent_rects.length; i++) { const a = transparent_rects[i]; - - if (a.img) { - this.drawImage(a.img, a.x, a.y, a.width, a.height); + let screenIndexes = this._getRectScreenIndexes(a); + + for (let sI = 0; sI < screenLocations.length; sI++) { + let screenLocation = a.screenLocations[sI]; + if (sI == 0) { + if (a.img) { + this.drawImage(a.img, a.x, a.y, a.width, a.height); + } + } else { + this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + } } } @@ -823,6 +942,41 @@ export default class Display { } } + _processRectScreens(rect) { + + //find which screen this rect belongs to and adjust its x and y to be relative to the destination + let indexes = []; + rect.inPrimary = false; + rect.inSecondary = false; + for (let i=0; i < this._screens.length; i++) { + let screen = this._screens[i]; + if ( + ((rect.x >= screen.x && rect.x < screen.x + screen.width) && + (rect.y >= screen.y && rect.y < screen.y + screen.height)) || + ((rect.x+rect.width >= screen.x && rect.x+rect.width < screen.x + screen.width) && + (rect.y+rect.height >= screen.y && rect.y+rect.height < screen.y + screen.height)) + ) { + let screenPosition = { + x: 0 - (screen.x - rect.x), //rect.x - screen.x, + y: 0 - (screen.y - rect.y), //rect.y - screen.y, + screenIndex: i + } + if (rect.type === 'copy') { + screenPosition.oldX = 0 - (screen.x - rect.oldX); //rect.oldX - screen.x; + screenPosition.oldY = 0 - (screen.y - rect.oldY); //rect.oldY - screen.y; + } + indexes.push(screenPosition); + if (i == 0) { + rect.inPrimary = true; + } else { + rect.inSecondary = true; + } + } + } + + rect.screenLocations = indexes; + } + _rescale(factor) { this._scale = factor; const vp = this._screens[0]; diff --git a/core/input/keyboard.js b/core/input/keyboard.js index db3c97ab..169c59b6 100644 --- a/core/input/keyboard.js +++ b/core/input/keyboard.js @@ -11,7 +11,6 @@ import KeyTable from "./keysym.js"; import keysyms from "./keysymdef.js"; import imekeys from "./imekeys.js"; import * as browser from "../util/browser.js"; -import UI from '../../app/ui.js'; import { isChromiumBased } from '../util/browser.js'; // @@ -46,6 +45,7 @@ export default class Keyboard { this._lastKeyboardInput = null; this._defaultKeyboardInputLen = 100; this._keyboardInputReset(); + this._translateShortcuts = true; } // ===== PUBLIC METHODS ===== @@ -56,6 +56,9 @@ export default class Keyboard { this.focus(); } + get translateShortcuts() { return this._translateShortcuts; } + set translateShortcuts(value) { this._translateShortcuts = value; } + // ===== PRIVATE METHODS ===== clearKeysDown(event) { @@ -319,7 +322,7 @@ export default class Keyboard { // Translate MacOs CMD based shortcuts to their CTRL based counterpart if ( browser.isMac() && - UI.rfb && UI.rfb.translateShortcuts && + this._translateShortcuts && code !== "MetaLeft" && code !== "MetaRight" && e.metaKey && !e.ctrlKey && !e.altKey ) { diff --git a/core/rfb.js b/core/rfb.js index bb6813a6..24c50818 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -80,7 +80,7 @@ export default class RFB extends EventTargetMixin { if (!target) { throw new Error("Must specify target"); } - if (!urlOrChannel) { + if (!urlOrChannel && isPrimaryDisplay) { throw new Error("Must specify URL, WebSocket or RTCDataChannel"); } @@ -219,7 +219,7 @@ export default class RFB extends EventTargetMixin { this._supportsBroadcastChannel = (typeof BroadcastChannel !== "undefined"); if (this._supportsBroadcastChannel) { this._controlChannel = new BroadcastChannel("registrationChannel"); - this._controlChannel.message = this._handleControlMessage.bind(this); + this._controlChannel.addEventListener('message', this._handleControlMessage.bind(this)); Log.Debug("Attached to registrationChannel for secondary displays.") } @@ -277,7 +277,7 @@ export default class RFB extends EventTargetMixin { // NB: nothing that needs explicit teardown should be done // before this point, since this can throw an exception try { - this._display = new Display(this._canvas); + this._display = new Display(this._canvas, this._isPrimaryDisplay); } catch (exc) { Log.Error("Display exception: " + exc); throw exc; @@ -300,6 +300,10 @@ export default class RFB extends EventTargetMixin { if (this._isPrimaryDisplay) { this._setupWebSocket(); + } else { + this._updateConnectionState('connecting'); + this._registerSecondaryDisplay(); + this._updateConnectionState('connected'); } Log.Debug("<< RFB.constructor"); @@ -327,6 +331,11 @@ export default class RFB extends EventTargetMixin { // ===== PROPERTIES ===== + get translateShortcuts() { return this._keyboard.translateShortcuts; } + set translateShortcuts(value) { + this._keyboard.translateShortcuts = value; + } + get pointerLock() { return this._pointerLock; } set pointerLock(value) { if (!this._pointerLock) { @@ -1065,8 +1074,6 @@ export default class RFB extends EventTargetMixin { } catch (e) { this._fail("Error attaching channel (" + e + ")"); } - } else { - this._registerSecondaryDisplay(); } // Make our elements part of the page @@ -1608,30 +1615,8 @@ export default class RFB extends EventTargetMixin { { detail: { capabilities: this._capabilities } })); } - _registerSecondaryDisplay() { - this._primaryDisplayChannel = new BroadcastChannel(`screen_${this._screenID}_channel`); - this._primaryDisplayChannel.message = this._handleSecondaryDisplayMessage.bind(this); - const size = this._screenSize(); - - message = { - eventType: 'register', - screenID: this._screenID, - screenIndex: this._screenIndex, - width: size.w, - height: size.h, - x: 0, - y: 0, - relativePosition: 0, - pixelRatio: window.devicePixelRatio, - containerWidth: this._screen.offsetWidth, - containerHeight: this._screen.offsetWidth, - channel: null - } - this._controlChannel.postMessage(message) - } - _proxyRFBMessage(messageType, data) { - message = { + let message = { messageType: messageType, data: data } @@ -1641,19 +1626,50 @@ export default class RFB extends EventTargetMixin { _handleControlMessage(event) { console.log(event); - switch (event.eventType) { - case 'register': - this._display.addScreen(event.screenID, event.width, event.height, event.relativePosition, event.pixelRatio, event.containerHeight, event.containerWidth); - Log.Info(`Secondary monitor (${event.screenID}) has been registered.`); - break; - case 'unregister': - if (this._display.removeScreen(event.screenID)) { - Log.Info(`Secondary monitor (${event.screenID}) has been removed.`); - } else { - Log.Info(`Secondary monitor (${event.screenID}) not found.`); - } + if (this._isPrimaryDisplay) { + switch (event.data.eventType) { + case 'register': + this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); + const size = this._screenSize(); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); + Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); + break; + case 'unregister': + if (this._display.removeScreen(event.data.screenID)) { + Log.Info(`Secondary monitor (${event.data.screenID}) has been removed.`); + const size = this._screenSize(); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); + } else { + Log.Info(`Secondary monitor (${event.data.screenID}) not found.`); + } + } } + + } + _registerSecondaryDisplay() { + if (!this._isPrimaryDisplay) { + let screen = this._screenSize().screens[0]; + this._display.resize(screen.containerWidth, screen.containerWidth); + screen = this._screenSize().screens[0]; + + + let message = { + eventType: 'register', + screenID: screen.screenID, + screenIndex: 1, + width: screen.width, + height: screen.height, + x: 0, + y: 0, + relativePosition: 0, + pixelRatio: screen.pixelRatio, + containerWidth: screen.containerWidth, + containerHeight: screen.containerHeight, + channel: null + } + this._controlChannel.postMessage(message); + } } From 05735088b7ca84fdc3f56a99fb7a591a56840844 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 13 Sep 2023 05:44:54 -0400 Subject: [PATCH 03/41] WIP - lots of fixes --- core/display.js | 65 ++++++++++++++++++++++++++++++++----------------- core/rfb.js | 9 ++++--- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/core/display.js b/core/display.js index fa003a66..1b648504 100644 --- a/core/display.js +++ b/core/display.js @@ -578,11 +578,11 @@ export default class Display { img.src = "data: " + mime + ";base64," + Base64.encode(arr); rect.img = img; } else { - rect.type = 'img_array' + rect.type = "_img"; } if (rect.inSecondary) { rect.mime = mime; - rect.arr = arr; + rect.src = "data: " + mime + ";base64," + Base64.encode(arr); } this._asyncRenderQPush(rect); @@ -706,43 +706,49 @@ export default class Display { switch (event.data.eventType) { case 'rect': let rect = event.data.rect; - let pos = rect.screenLocations[event.data.screenLocationIndex]; - if (!pos) { - console.log('wtf'); - } + //overwrite screen locations when received on the secondary display + rect.screenLocations = [ rect.screenLocations[event.data.screenLocationIndex] ] + rect.screenLocations[0].screenIndex = 0; + let pos = rect.screenLocations[0]; + switch (rect.type) { case 'copy': - this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); + //this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); + console.log(`Copy Rect: src.x: ${rect.oldX}, src.y: ${rect.oldY}, x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`) + this._asyncRenderQPush(rect); break; case 'fill': - this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); + this._asyncRenderQPush(rect); + //this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); break; case 'blit': - this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + this._asyncRenderQPush(rect); + //this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'blitQ': - this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + this._asyncRenderQPush(rect); + //this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'img': - case 'img_arr': + case '_img': rect.img = new Image(); - rect.img.src = "data: " + rect.mime + ";base64," + Base64.encode(rect.arr); - if (!rect.img.complete) { - rect.img.addEventListener('load', function (rect) { - this.drawImage(rect.img, pos.x, pos.y, rect.width, rect.height); - }.bind(this, rect)); - } else { - this.drawImage(rect.img, pos.x, pos.y, rect.width, rect.height); - } + rect.img.src = rect.src; + rect.type = 'img'; + this._asyncRenderQPush(rect); break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); imageBmpPromise.then(function(rect, img) { - this.drawImage(img, pos.x, pos.y, rect.width, rect.height); + rect.img.complete = true; }).bind(this, rect); + this._asyncRenderQPush(rect); break; } break; + case 'frameComplete': + this.flip(event.data.frameId, event.data.rectCnt); + + break; } } } @@ -783,7 +789,7 @@ export default class Display { } } - if (this._asyncFrameQueue[frameIx][1] == this._asyncFrameQueue[frameIx][2].length) { + if (this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { //frame is complete this._asyncFrameComplete(frameIx); } @@ -795,6 +801,7 @@ export default class Display { return; } else if (rect.frame_id > newestFrameID) { //frame is newer than any frame in the queue, drop old frames + Log.Warn("Older Rect Dropped"); this._asyncFrameQueue.shift(); let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0); this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0, 0 ]); @@ -861,12 +868,14 @@ export default class Display { */ _pushAsyncFrame(force=false) { if (this._asyncFrameQueue[0][3] || force) { - let frame = this._asyncFrameQueue.shift()[2]; + let frame = this._asyncFrameQueue[0][2]; + let frameId = this._asyncFrameQueue.shift()[0]; if (this._asyncFrameQueue.length < this._maxAsyncFrameQueue) { this._asyncFrameQueue.push([ 0, 0, [], false, 0, 0 ]); } let transparent_rects = []; + let secondaryScreenRects = 0; //render the selected frame for (let i = 0; i < frame.length; i++) { @@ -900,7 +909,10 @@ export default class Display { if (a.img) { a.img = null; } - this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + if (a.type !== 'flip') { + secondaryScreenRects++; + this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); + } } } } @@ -917,11 +929,18 @@ export default class Display { this.drawImage(a.img, a.x, a.y, a.width, a.height); } } else { + secondaryScreenRects++; this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); } } } + if (secondaryScreenRects > 0) { + for (let i = 1; i < this.screens.length; i++) { + this._screens[i].channel.postMessage({ eventType: 'frameComplete', frameId: frameId, rectCnt: secondaryScreenRects }); + } + } + this._flipCnt += 1; if (this._flushing) { diff --git a/core/rfb.js b/core/rfb.js index 24c50818..bf114a78 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1624,8 +1624,6 @@ export default class RFB extends EventTargetMixin { } _handleControlMessage(event) { - console.log(event); - if (this._isPrimaryDisplay) { switch (event.data.eventType) { case 'register': @@ -1649,8 +1647,11 @@ export default class RFB extends EventTargetMixin { _registerSecondaryDisplay() { if (!this._isPrimaryDisplay) { - let screen = this._screenSize().screens[0]; - this._display.resize(screen.containerWidth, screen.containerWidth); + //let screen = this._screenSize().screens[0]; + // + let size = this._screenSize(); + this._display.resize(size.screens[0].containerWidth, size.screens[0].containerHeight); + this._display.autoscale(size.screens[0].containerWidth, size.screens[0].containerHeight, size.screens[0].scale); screen = this._screenSize().screens[0]; From cb1bc7f7874703e4cb4679d3ab1efb953caf6db2 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 20 Sep 2023 05:33:40 -0400 Subject: [PATCH 04/41] wired up mouse, keyboard, clipboard on second display --- app/ui.js | 2 +- app/ui_screen.js | 26 ++++++++++++------- core/display.js | 67 +++++++++++++++++++++++++++++++++++++++++++++--- core/rfb.js | 36 +++++++++++++++++++++++--- 4 files changed, 115 insertions(+), 16 deletions(-) diff --git a/app/ui.js b/app/ui.js index bc1ed064..f866bd3c 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1421,7 +1421,7 @@ const UI = { UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper(); if (UI.rfb.videoQuality === 5) { UI.rfb.enableQOI = true; - } + } //Only explicitly request permission to clipboard on browsers that support binary clipboard access if (supportsBinaryClipboard()) { diff --git a/app/ui_screen.js b/app/ui_screen.js index b772e39f..4ec64a02 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -3,6 +3,7 @@ import * as WebUtil from "./webutil.js"; import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS, supportsPointerLock } from '../core/util/browser.js'; import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js"; +import * as Log from '../core/util/logging.js'; const UI = { connected: false, @@ -29,9 +30,11 @@ const UI = { document.getElementById('noVNC_connect_button').addEventListener('click', UI.connect);; }, - getSetting(name, isBool) { - const ctrl = document.getElementById('noVNC_setting_' + name); + getSetting(name, isBool, default_value) { let val = WebUtil.readSetting(name); + if ((val === 'undefined' || val === null) && default_value !== 'undefined' && default_value !== null) { + val = default_value; + } if (typeof val !== 'undefined' && val !== null && isBool) { if (val.toString().toLowerCase() in {'0': 1, 'no': 1, 'false': 1}) { val = false; @@ -72,22 +75,27 @@ const UI = { UI.rfb.maxVideoResolutionY = parseInt(UI.getSetting('max_video_resolution_y')); UI.rfb.frameRate = parseInt(UI.getSetting('framerate')); UI.rfb.compressionLevel = parseInt(UI.getSetting('compression')); - UI.rfb.showDotCursor = UI.getSetting('show_dot'); + UI.rfb.showDotCursor = UI.getSetting('show_dot', true); UI.rfb.idleDisconnect = UI.getSetting('idle_disconnect'); UI.rfb.pointerRelative = UI.getSetting('pointer_relative'); UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality')); UI.rfb.antiAliasing = UI.getSetting('anti_aliasing'); - UI.rfb.clipboardUp = UI.getSetting('clipboard_up'); - UI.rfb.clipboardDown = UI.getSetting('clipboard_down'); - UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless'); - UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime'); + UI.rfb.clipboardUp = UI.getSetting('clipboard_up', true, true); + UI.rfb.clipboardDown = UI.getSetting('clipboard_down', true, true); + UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless', true, true); + UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime', true, false); UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless; - UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc'); - UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi'); + UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc', true, false); + UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi', true, false); UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper(); if (UI.rfb.videoQuality === 5) { UI.rfb.enableQOI = true; } + + if (supportsBinaryClipboard()) { + // explicitly request permission to the clipboard + navigator.permissions.query({ name: "clipboard-read" }).then((result) => { Log.Debug('binary clipboard enabled') }); + } }, updateVisualState(state) { diff --git a/core/display.js b/core/display.js index 1b648504..ad21264c 100644 --- a/core/display.js +++ b/core/display.js @@ -121,6 +121,7 @@ export default class Display { // ===== PROPERTIES ===== get screens() { return this._screens; } + get screenId() { return this._screenID; } get antiAliasing() { return this._antiAliasing; } set antiAliasing(value) { @@ -161,6 +162,66 @@ export default class Display { // ===== PUBLIC METHODS ===== + /* + Returns coordinates that are client relative when multiple monitors are in use + Returns an array with the following + 0 - screen index + 1 - screenId + 2 - x + 3 - y + */ + getClientRelativeCoordinates(x, y) { + if (this._screens.length == 1) { + return [ 0, this._screenID, x, y ]; + } + //TODO: The following logic will only support two monitors placed horizontally + let screenOrientation = this._screens[1].relativePosition; + let screenIdx = 0; + let screenId = this._screens[0].screenID; + if (screenOrientation == 0) { + if (x >= this._screens[1].x) { + x -= this._screens[1].x; + screenIdx = 1; + screenId = this._screens[1].screenID; + } + } else if (screenOrientation == 2) { + if (x >= this._screens[0].x) { + x -= this._screens[0].x; + } + } + return [ screenIdx, screenId, x, y ]; + } + + /* + Returns coordinates that are server relative when multiple monitors are in use + */ + getServerRelativeCoordinates(screenId, x, y) { + // If this is the primary screen and only one screen, lets keep it simple + if (this._isPrimaryDisplay && this._screens.length == 1) { + return [x, y]; + } + + // Find the screen index by ID + let screenIdx = -1; + for (let i=0; i Date: Wed, 20 Sep 2023 08:17:05 -0400 Subject: [PATCH 05/41] Added support for all orientations and offsets --- app/ui_screen.js | 19 ++++++++ core/display.js | 118 ++++++++++++++++++++++++++++++++--------------- core/rfb.js | 22 ++++++--- 3 files changed, 116 insertions(+), 43 deletions(-) diff --git a/app/ui_screen.js b/app/ui_screen.js index 4ec64a02..85d24f07 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -92,6 +92,20 @@ const UI = { UI.rfb.enableQOI = true; } + // attach secondary display with relative position, relative x, and relative y + // relativePosition: + // 0: primary display is to left + // 1: primary display is up top + // 2: primary display is to right + // 3: primary display is down below + // relativePositionX: + // non-zero number only allowed if relativePosition is 1 or 3 + // How many pixels on the X axis is the secondary screens starting position from the primary displays + // relativePositionY: + // non-zero number only allowed if relativePosition is 0 or 2 + // How many pixels on the Y axis is the secondary screens starting position from the primary displays + UI.rfb.attachSecondaryDisplay(3, 0, 0); + if (supportsBinaryClipboard()) { // explicitly request permission to the clipboard navigator.permissions.query({ name: "clipboard-read" }).then((result) => { Log.Debug('binary clipboard enabled') }); @@ -200,6 +214,11 @@ const UI = { } }, + hideStatus() { + clearTimeout(UI.statusTimeout); + document.getElementById('noVNC_status').classList.remove("noVNC_open"); + }, + disconnect() { }, diff --git a/core/display.js b/core/display.js index ad21264c..269ccca2 100644 --- a/core/display.js +++ b/core/display.js @@ -97,6 +97,8 @@ export default class Display { x: 0, y: 0, relativePosition: 0, + relativePositionX: 0, + relativePositionY: 0, pixelRatio: window.devicePixelRatio, containerHeight: this._target.parentNode.offsetHeight, containerWidth: this._target.parentNode.offsetWidth, @@ -120,6 +122,30 @@ export default class Display { // ===== PROPERTIES ===== + get relativePosition() { return this._screens[0].relativePosition; } + set relativePosition(value) { + if (!this._isPrimaryDisplay && value >= 0 && value < 4) { + this._screens[0].relativePosition = value; + //reset relative X and Y + this._screens[0].relativePositionX = 0; + this._screens[0].relativePositionY = 0; + } + } + + get relativePositionX() { return this._screens[0].relativePositionX; } + set relativePositionX(value) { + if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 1 || this._screens[0].relativePosition == 3)) { + this._screens[0].relativePositionX = value; + } + } + + get relativePositionY() { return this._screens[0].relativePositionY; } + set relativePositionY(value) { + if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 0 || this._screens[0].relativePosition == 2)) { + this._screens[0].relativePositionY = value; + } + } + get screens() { return this._screens; } get screenId() { return this._screenID; } @@ -226,9 +252,7 @@ export default class Display { let data = { screens: null, serverWidth: 0, - serverHeight: 0, - clientWidth: 0, - clientHeight: 0 + serverHeight: 0 } //recalculate primary display container size @@ -274,54 +298,72 @@ export default class Display { this._screens[i].serverWidth = width; this._screens[i].serverHeight = height; - - //this logic will only support monitors laid out side by side - //TODO: two vertically stacked monitors would require light refactoring here - data.serverWidth += width; - data.serverHeight = Math.max(data.serverHeight, height); - data.clientWidth += this._screens[i].width; - data.clientHeight = Math.max(data.clientHeight, this._screens[i].height); this._screens[i].scale = scale; } - //calculate positions of monitors, this logic will only support two monitors side by side in either order + const primary_screen = this._screens[0]; + //reset primary display position + primary_screen.x = 0; + primary_screen.y = 0; + let total_server_width = primary_screen.serverWidth; + let total_server_height = primary_screen.serverHeight; + + //TODO: The following logic will only support two monitors + // Calculate total area of all screens and positions of each screen within the total area if (this._screens.length > 1) { - const primary_screen = this._screens[0]; + const secondary_screen = this._screens[1]; - let total_width = 0; - let total_height = 0; + secondary_screen.x = 0; + secondary_screen.y = 0; switch (this._screens[1].relativePosition) { case 0: //primary screen is to left - total_width = secondary_screen.serverWidth + primary_screen.serverWidth; - total_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight); + total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; + total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); secondary_screen.x = primary_screen.serverWidth; - if (secondary_screen.serverHeight < primary_screen.serverHeight) { - if ((total_height - secondary_screen.serverHeight) > 1) { - secondary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) - } + + if (secondary_screen.relativePositionY >= 0) { + secondary_screen.y = secondary_screen.relativePositionY; } else { - secondary_screen.y = 0; - if ((total_height - secondary_screen.serverHeight) > 1) { - this._screens[0].y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) - } + primary_screen.y = Math.abs(secondary_screen.relativePositionY); + } + + break; + case 1: + //primary screen is up above + total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); + total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; + secondary_screen.y = primary_screen.serverHeight; + + if (secondary_screen.relativePositionX >= 0) { + secondary_screen.x = secondary_screen.relativePositionX; + } else { + primary_screen.x = Math.abs(secondary_screen.relativePositionX); } break; case 2: //primary screen is to right - total_width = primary_screen.serverWidth + secondary_screen.serverWidth; - total_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight); - this._screens[0].x = secondary_screen.serverWidth; - if (secondary_screen.serverHeight < primary_screen.serverHeight) { - if ((total_height - secondary_screen.serverHeight) > 1) { - secondary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) - } + total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; + total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); + primary_screen.x = secondary_screen.serverWidth; + + if (secondary_screen.relativePositionY >= 0) { + secondary_screen.y = secondary_screen.relativePositionY; } else { - secondary_screen.y = 0; - if ((total_height - secondary_screen.serverHeight) > 1) { - primary_screen.y = Math.abs(Math.round(((total_height - secondary_screen.serverHeight) / 2))) - } + primary_screen.y = Math.abs(secondary_screen.relativePositionY); + } + break; + case 3: + //primary screen is down below + total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); + total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; + primary_screen.y = secondary_screen.serverHeight; + + if (secondary_screen.relativePositionX >= 0) { + secondary_screen.x = secondary_screen.relativePositionX; + } else { + primary_screen.x = Math.abs(secondary_screen.relativePositionX); } break; default: @@ -331,11 +373,13 @@ export default class Display { } data.screens = this._screens; + data.serverWidth = total_server_width; + data.serverHeight = total_server_height; return data; } - addScreen(screenID, width, height, relativePosition, pixelRatio, containerHeight, containerWidth) { + addScreen(screenID, width, height, relativePosition, relativePositionX, relativePositionY, pixelRatio, containerHeight, containerWidth) { if (this._isPrimaryDisplay) { //currently only support one secondary screen if (this._screens.length > 1) { @@ -353,6 +397,8 @@ export default class Display { x: 0, y: 0, relativePosition: relativePosition, + relativePositionX: relativePositionX, + relativePositionY: relativePositionY, pixelRatio: pixelRatio, containerHeight: containerHeight, containerWidth: containerWidth, diff --git a/core/rfb.js b/core/rfb.js index 313115b5..210911a9 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -300,11 +300,7 @@ export default class RFB extends EventTargetMixin { if (this._isPrimaryDisplay) { this._setupWebSocket(); - } else { - this._updateConnectionState('connecting'); - this._registerSecondaryDisplay(); - this._updateConnectionState('connected'); - } + } Log.Debug("<< RFB.constructor"); @@ -736,6 +732,16 @@ export default class RFB extends EventTargetMixin { // ===== PUBLIC METHODS ===== + attachSecondaryDisplay(relativePosition, relativePositionX, relativePositionY) { + this._display.relativePosition = relativePosition; + this._display.relativePositionX = relativePositionX; + this._display.relativePositionY = relativePositionY; + + this._updateConnectionState('connecting'); + this._registerSecondaryDisplay(); + this._updateConnectionState('connected'); + } + /* This function must be called after changing any properties that effect rendering quality */ @@ -1629,7 +1635,7 @@ export default class RFB extends EventTargetMixin { // Secondary to Primary screen message switch (event.data.eventType) { case 'register': - this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); + this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.relativePositionX, event.data.relativePositionY, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); const size = this._screenSize(); RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); this._updateContinuousUpdates(); @@ -1688,7 +1694,9 @@ export default class RFB extends EventTargetMixin { height: screen.height, x: 0, y: 0, - relativePosition: 0, + relativePosition: screen.relativePosition, + relativePositionX: screen.relativePositionX, + relativePositionY: screen.relativePositionY, pixelRatio: screen.pixelRatio, containerWidth: screen.containerWidth, containerHeight: screen.containerHeight, From 23076cdc7c57384c7ec302e337b28e1e41553b4b Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 20 Sep 2023 09:06:27 -0400 Subject: [PATCH 06/41] unregister a display if window closes --- app/ui_screen.js | 4 +++- core/display.js | 3 ++- core/rfb.js | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/app/ui_screen.js b/app/ui_screen.js index 85d24f07..d66a9416 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -220,7 +220,9 @@ const UI = { }, disconnect() { - + if (UI.rfb) { + UI.rfb.disconnect(); + } }, connectFinished(e) { diff --git a/core/display.js b/core/display.js index 269ccca2..6b690b86 100644 --- a/core/display.js +++ b/core/display.js @@ -418,9 +418,10 @@ export default class Display { } removeScreen(screenID) { - if (this.isPrimaryDisplay) { + if (this._isPrimaryDisplay) { for (let i=1; i Date: Wed, 20 Sep 2023 15:18:08 -0400 Subject: [PATCH 07/41] refactor to support any number of displays in any orientation --- app/ui.js | 14 +++ app/ui_screen.js | 15 +--- core/display.js | 215 +++++++++++++---------------------------------- core/rfb.js | 97 ++++++++++++++++----- 4 files changed, 149 insertions(+), 192 deletions(-) diff --git a/app/ui.js b/app/ui.js index f866bd3c..5cc0e697 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1388,6 +1388,7 @@ const UI = { UI.rfb.addEventListener("desktopname", UI.updateDesktopName); UI.rfb.addEventListener("inputlock", UI.inputLockChanged); UI.rfb.addEventListener("inputlockerror", UI.inputLockError); + UI.rfb.addEventListener("screenregistered", UI.screenRegistered); UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts'); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; @@ -2471,6 +2472,19 @@ const UI = { } }, + screenRegistered(e) { + // Get the current screen plan + // When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top + let screenPlan = UI.rfb.getScreenPlan(); + + // Now make adjustments to the screen plan, this is just an example + screenPlan.screens[1].y = 100; + + // Finally apply the screen plan + UI.rfb.applyScreenPlan(screenPlan); + console.log(screenPlan); + }, + //Helper to add options to dropdown. addOption(selectbox, text, value) { const optn = document.createElement("OPTION"); diff --git a/app/ui_screen.js b/app/ui_screen.js index d66a9416..5e193979 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -92,19 +92,8 @@ const UI = { UI.rfb.enableQOI = true; } - // attach secondary display with relative position, relative x, and relative y - // relativePosition: - // 0: primary display is to left - // 1: primary display is up top - // 2: primary display is to right - // 3: primary display is down below - // relativePositionX: - // non-zero number only allowed if relativePosition is 1 or 3 - // How many pixels on the X axis is the secondary screens starting position from the primary displays - // relativePositionY: - // non-zero number only allowed if relativePosition is 0 or 2 - // How many pixels on the Y axis is the secondary screens starting position from the primary displays - UI.rfb.attachSecondaryDisplay(3, 0, 0); + //attach this secondary display to the primary display + UI.rfb.attachSecondaryDisplay(); if (supportsBinaryClipboard()) { // explicitly request permission to the clipboard diff --git a/core/display.js b/core/display.js index 6b690b86..ab7b4b80 100644 --- a/core/display.js +++ b/core/display.js @@ -11,7 +11,7 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; -import { uuidv4 } from './util/strings.js' +import { uuidv4 } from './util/strings.js'; export default class Display { constructor(target, isPrimaryDisplay) { @@ -81,6 +81,7 @@ export default class Display { // ===== PROPERTIES ===== + this._maxScreens = 4; this._scale = 1.0; this._clipViewport = false; this._antiAliasing = 0; @@ -96,9 +97,9 @@ export default class Display { serverHeight: 0, //calculated x: 0, y: 0, - relativePosition: 0, - relativePositionX: 0, - relativePositionY: 0, + relativePosition: 0, //left, right, up, down relative to primary display + relativePositionX: 0, //offset relative to primary monitor, always 0 for primary + relativePositionY: 0, //offset relative to primary monitor, always 0 for primary pixelRatio: window.devicePixelRatio, containerHeight: this._target.parentNode.offsetHeight, containerWidth: this._target.parentNode.offsetWidth, @@ -122,32 +123,16 @@ export default class Display { // ===== PROPERTIES ===== - get relativePosition() { return this._screens[0].relativePosition; } - set relativePosition(value) { - if (!this._isPrimaryDisplay && value >= 0 && value < 4) { - this._screens[0].relativePosition = value; - //reset relative X and Y - this._screens[0].relativePositionX = 0; - this._screens[0].relativePositionY = 0; - } - } - - get relativePositionX() { return this._screens[0].relativePositionX; } - set relativePositionX(value) { - if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 1 || this._screens[0].relativePosition == 3)) { - this._screens[0].relativePositionX = value; - } - } - - get relativePositionY() { return this._screens[0].relativePositionY; } - set relativePositionY(value) { - if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 0 || this._screens[0].relativePosition == 2)) { - this._screens[0].relativePositionY = value; - } - } - get screens() { return this._screens; } get screenId() { return this._screenID; } + get screenIndex() { + // A secondary screen should not have a screen index of 0, but it will be 0 until registration is complete + // returning a -1 lets the caller know the screen has not been registered yet + if (!this._isPrimaryDisplay && this._screens[0].screenIndex == 0) { + return -1; + } + return this._screens[0].screenIndex; + } get antiAliasing() { return this._antiAliasing; } set antiAliasing(value) { @@ -189,62 +174,21 @@ export default class Display { // ===== PUBLIC METHODS ===== /* - Returns coordinates that are client relative when multiple monitors are in use - Returns an array with the following - 0 - screen index - 1 - screenId - 2 - x - 3 - y + Returns the screen index given serverside relative coordinates */ - getClientRelativeCoordinates(x, y) { - if (this._screens.length == 1) { - return [ 0, this._screenID, x, y ]; - } - //TODO: The following logic will only support two monitors placed horizontally - let screenOrientation = this._screens[1].relativePosition; - let screenIdx = 0; - let screenId = this._screens[0].screenID; - if (screenOrientation == 0) { - if (x >= this._screens[1].x) { - x -= this._screens[1].x; - screenIdx = 1; - screenId = this._screens[1].screenID; - } - } else if (screenOrientation == 2) { - if (x >= this._screens[0].x) { - x -= this._screens[0].x; - } - } - return [ screenIdx, screenId, x, y ]; + getScreenIndexByServerCoords(x, y) { + } /* Returns coordinates that are server relative when multiple monitors are in use */ - getServerRelativeCoordinates(screenId, x, y) { - // If this is the primary screen and only one screen, lets keep it simple - if (this._isPrimaryDisplay && this._screens.length == 1) { - return [x, y]; + getServerRelativeCoordinates(screenIndex, x, y) { + if (screenIndex >= 0 && screenIndex < this._screens.length) { + x += this._screens[screenIndex].x; + y += this._screens[screenIndex].y; } - // Find the screen index by ID - let screenIdx = -1; - for (let i=0; i 1) { - - const secondary_screen = this._screens[1]; - secondary_screen.x = 0; - secondary_screen.y = 0; - - switch (this._screens[1].relativePosition) { - case 0: - //primary screen is to left - total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; - total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); - secondary_screen.x = primary_screen.serverWidth; - - if (secondary_screen.relativePositionY >= 0) { - secondary_screen.y = secondary_screen.relativePositionY; - } else { - primary_screen.y = Math.abs(secondary_screen.relativePositionY); - } - - break; - case 1: - //primary screen is up above - total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); - total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; - secondary_screen.y = primary_screen.serverHeight; - - if (secondary_screen.relativePositionX >= 0) { - secondary_screen.x = secondary_screen.relativePositionX; - } else { - primary_screen.x = Math.abs(secondary_screen.relativePositionX); - } - break; - case 2: - //primary screen is to right - total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; - total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); - primary_screen.x = secondary_screen.serverWidth; - - if (secondary_screen.relativePositionY >= 0) { - secondary_screen.y = secondary_screen.relativePositionY; - } else { - primary_screen.y = Math.abs(secondary_screen.relativePositionY); - } - break; - case 3: - //primary screen is down below - total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); - total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; - primary_screen.y = secondary_screen.serverHeight; - - if (secondary_screen.relativePositionX >= 0) { - secondary_screen.x = secondary_screen.relativePositionX; - } else { - primary_screen.x = Math.abs(secondary_screen.relativePositionX); - } - break; - default: - //TODO: It would not be hard to support vertically stacked monitors - throw new Error("Unsupported screen orientation."); - } + for (let i = 0; i < this._screens.length; i++) { + data.serverWidth = Math.max(data.serverWidth, this._screens[i].x + this._screens[i].width); + data.serverHeight = Math.max(data.serverHeight, this._screens[i].y + this._screens[i].height); } data.screens = this._screens; - data.serverWidth = total_server_width; - data.serverHeight = total_server_height; return data; } - addScreen(screenID, width, height, relativePosition, relativePositionX, relativePositionY, pixelRatio, containerHeight, containerWidth) { + applyScreenPlan(screenPlan) { + for (let i = 0; i < screenPlan.screens.length; i++) { + for (let z = 0; z < this._screens.length; z++) { + if (screenPlan.screens[i].screenID === this._screens[z].screenID) { + this._screens[z].x = screenPlan.screens[i].x; + this._screens[z].y = screenPlan.screens[i].y; + } + } + } + } + + addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth) { if (this._isPrimaryDisplay) { - //currently only support one secondary screen - if (this._screens.length > 1) { - this._screens[1].channel.close(); - this._screens.pop() + //for now, place new screen to the far right, until the user repositions it + let x = 0; + for (let i = 0; i < this._screens.length; i++) { + x = Math.max(x, this._screens[i].x + this._screens[i].width); } var new_screen = { @@ -394,11 +281,8 @@ export default class Display { height: height, //client serverWidth: 0, //calculated serverHeight: 0, //calculated - x: 0, + x: x, y: 0, - relativePosition: relativePosition, - relativePositionX: relativePositionX, - relativePositionY: relativePositionY, pixelRatio: pixelRatio, containerHeight: containerHeight, containerWidth: containerWidth, @@ -418,15 +302,24 @@ export default class Display { } removeScreen(screenID) { + let removed = false; if (this._isPrimaryDisplay) { for (let i=1; i 0) { + this._screens[i].channel.postMessage({ eventType: "registered", screenIndex: i }); + } + } + return removed; } else { throw new Error("Secondary screens only allowed on primary display.") } @@ -857,6 +750,12 @@ export default class Display { this.flip(event.data.frameId, event.data.rectCnt); break; + case 'registered': + if (!this._isPrimaryDisplay) { + this._screens[0].screenIndex = event.data.screenIndex; + Log.Info(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`); + } + break; } } } diff --git a/core/rfb.js b/core/rfb.js index 13a83001..42d6e4b8 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -207,6 +207,7 @@ export default class RFB extends EventTargetMixin { this._accumulatedWheelDeltaX = 0; this._accumulatedWheelDeltaY = 0; this.mouseButtonMapper = null; + this._mouseLastScreenIndex = 0; // Gesture state this._gestureLastTapTime = null; @@ -732,16 +733,68 @@ export default class RFB extends EventTargetMixin { // ===== PUBLIC METHODS ===== - attachSecondaryDisplay(relativePosition, relativePositionX, relativePositionY) { - this._display.relativePosition = relativePosition; - this._display.relativePositionX = relativePositionX; - this._display.relativePositionY = relativePositionY; - + attachSecondaryDisplay() { this._updateConnectionState('connecting'); this._registerSecondaryDisplay(); this._updateConnectionState('connected'); } + applyScreenPlan(screenPlan) { + if (this._isPrimaryDisplay) { + let fullPlan = this._display.getScreenSize(); + + //check plan for validity + let minX = Number.MAX_SAFE_INTEGER, minY = Number.MAX_SAFE_INTEGER; + let numScreensFound = 0; + + for (let i = 0; i < screenPlan.screens.length; i++) { + minX = Math.min(minX, screenPlan.screens[i].x); + minY = Math.min(minY, screenPlan.screens[i].y); + for (let z = 0; z < fullPlan.screens.length; z++) { + if (screenPlan.screens[i].screenID == fullPlan.screens[z].screenID) { + numScreensFound++; + } + } + } + if (minX !== 0 || minY !== 0) { + throw new Error("Screen plan invalid, improper coordinates provided."); + } + if (numScreensFound > fullPlan.screens.length) { + throw new Error("Screen plan contained more screens then there are registered.") + } else if (numScreensFound < fullPlan.screens.length) { + throw new Error("Screen plan contained fewer screens then there are registered.") + } + + this._display.applyScreenPlan(screenPlan); + const size = this._screenSize(); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); + this._updateContinuousUpdates(); + } + } + + getScreenPlan() { + let fullPlan = this._display.getScreenSize(); + let sanitizedPlan = { + screens: [], + serverWidth: fullPlan.serverWidth, + serverHeight: fullPlan.serverHeight + }; + + for (let i=0; i < fullPlan.screens.length; i++) { + sanitizedPlan.screens.push( + { + screenID: fullPlan.screens[i].screenID, + serverWidth: fullPlan.screens[i].serverWidth, + serverHeight: fullPlan.screens[i].serverHeight, + x: fullPlan.screens[i].x, + y: fullPlan.screens[i].y + } + ) + } + + return sanitizedPlan; + } + /* This function must be called after changing any properties that effect rendering quality */ @@ -1628,7 +1681,9 @@ export default class RFB extends EventTargetMixin { let message = { eventType: messageType, args: data, - screenId: this._display.screenId + screenId: this._display.screenId, + screenIndex: this._display.screenIndex, + mouseLastScreenIndex: this._mouseLastScreenIndex, } this._controlChannel.postMessage(message); } @@ -1638,10 +1693,11 @@ export default class RFB extends EventTargetMixin { // Secondary to Primary screen message switch (event.data.eventType) { case 'register': - this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.relativePositionX, event.data.relativePositionY, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); + this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); const size = this._screenSize(); RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); this._updateContinuousUpdates(); + this.dispatchEvent(new CustomEvent("screenregistered", {})); Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); break; case 'unregister': @@ -1654,9 +1710,11 @@ export default class RFB extends EventTargetMixin { } break; case 'pointerEvent': - let coords = this._display.getServerRelativeCoordinates(event.data.screenId, event.data.args[0], event.data.args[1]); + let coords = this._display.getServerRelativeCoordinates(event.data.screenIndex, event.data.args[0], event.data.args[1]); + this._mouseLastScreenIndex = event.data.screenIndex; event.data.args[0] = coords[0]; event.data.args[1] = coords[1]; + console.log(`screenIndex ${event.data.screenIndex}, x: ${coords[0]}, y: ${coords[1]}`); RFB.messages.pointerEvent(this._sock, ...event.data.args); break; case 'keyEvent': @@ -1665,6 +1723,9 @@ export default class RFB extends EventTargetMixin { case 'sendBinaryClipboard': RFB.messages.sendBinaryClipboard(this._sock, ...event.data.args); break; + // The following are primary to secondary messages that should be ignored on the primary + case 'updateCursor': + break; default: Log.Warn(`Unhandled message type (${event.data.eventType}) from control channel.`); } @@ -1672,7 +1733,9 @@ export default class RFB extends EventTargetMixin { // Primary to secondary screen message switch (event.data.eventType) { case 'updateCursor': - this._updateCursor(...event.data.args); + if (event.data.mouseLastScreenIndex === this._display.screenIndex) { + this._updateCursor(...event.data.args); + } break; } } @@ -1703,14 +1766,10 @@ export default class RFB extends EventTargetMixin { let message = { eventType: 'register', screenID: screen.screenID, - screenIndex: 1, width: screen.width, height: screen.height, x: 0, y: 0, - relativePosition: screen.relativePosition, - relativePositionX: screen.relativePositionX, - relativePositionY: screen.relativePositionY, pixelRatio: screen.pixelRatio, containerWidth: screen.containerWidth, containerHeight: screen.containerHeight, @@ -1818,6 +1877,7 @@ export default class RFB extends EventTargetMixin { this._canvas); } + this._mouseLastScreenIndex = this._display.screenIndex; this._setLastActive(); const mappedButton = this.mouseButtonMapper.get(ev.button); switch (ev.type) { @@ -1967,16 +2027,12 @@ export default class RFB extends EventTargetMixin { var rel_16_x = toSignedRelative16bit(x - this._pointerLockPos.x); var rel_16_y = toSignedRelative16bit(y - this._pointerLockPos.y); - //console.log("new_pos x" + x + ", y" + y); - //console.log("lock x " + this._pointerLockPos.x + ", y " + this._pointerLockPos.y); - //console.log("rel x " + rel_16_x + ", y " + rel_16_y); if (this._isPrimaryDisplay){ RFB.messages.pointerEvent(this._sock, rel_16_x, rel_16_y, mask); } else { this._proxyRFBMessage('pointerEvent', [ rel_16_x, rel_16_y, mask ]); } - // reset the cursor position to center this._mousePos = { x: this._pointerLockPos.x , y: this._pointerLockPos.y }; this._cursor.move(this._pointerLockPos.x, this._pointerLockPos.y); @@ -3436,9 +3492,9 @@ export default class RFB extends EventTargetMixin { const payload = this._sock.rQshiftStr(len); if (status) { - console.log("Unix relay subscription succeeded"); + Log.Info("Unix relay subscription succeeded"); } else { - console.log("Unix relay subscription failed, " + payload); + Log.Warn("Unix relay subscription failed, " + payload); } } @@ -3655,8 +3711,6 @@ export default class RFB extends EventTargetMixin { return false; } - console.log(`VMCursorUpdate x: ${hotx}, y: ${hoty}`); - this._updateCursor(rgba, hotx, hoty, w, h); return true; @@ -3881,6 +3935,7 @@ export default class RFB extends EventTargetMixin { rgbaPixels: rgba, hotx: hotx, hoty: hoty, w: w, h: h, }; + this._refreshCursor(); this._proxyRFBMessage('updateCursor', [ rgba, hotx, hoty, w, h ]); } From 1873ec0c91a9268243d3ad9800bbf2f09b15831c Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Fri, 29 Sep 2023 13:09:52 +0000 Subject: [PATCH 08/41] KASM-5078 Multi monitor display ui --- app/images/desktop-regular.svg | 1 + app/styles/base.css | 119 +++++++++++++ app/ui.js | 303 ++++++++++++++++++++++++++++++++- vnc.html | 26 ++- 4 files changed, 439 insertions(+), 10 deletions(-) create mode 100644 app/images/desktop-regular.svg diff --git a/app/images/desktop-regular.svg b/app/images/desktop-regular.svg new file mode 100644 index 00000000..cd265db1 --- /dev/null +++ b/app/images/desktop-regular.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/styles/base.css b/app/styles/base.css index 5a71fdc8..c45ca2ca 100644 --- a/app/styles/base.css +++ b/app/styles/base.css @@ -1191,3 +1191,122 @@ a:link { a:visited { color: white; } + +/* ---------------------------------------- + * Multi Display Arrangement + * ---------------------------------------- + */ + +.flex { + display: flex; +} +.flex-col { + flex-direction: column; +} +.canvas { + background: rgb(225,225,226); + background: linear-gradient(0deg, rgba(225,225,226,1) 0%, rgba(237,237,238,1) 100%); +} +.row { + background: #e9e9ea; + border-radius: 8px; + border: 1px solid #dcdcdd; + padding: 6px 15px; + display: flex; + justify-content: space-between; + align-items: center; +} +.row input { + border: 1px solid #dcdcdd; + padding: 8px 2px; + text-align: right; + background-color: #ededee; +} + + #noVNC_displays { + position: fixed; + display: none; + top: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 100; + background: rgba(0,0,0,0.7); + + cursor: pointer; + + transition: 0.5s ease-in-out; + + + padding: 5px; + flex-direction: row; + justify-content: center; + align-items: center; + + line-height: 25px; + word-wrap: break-word; + +} +#noVNC_displays.noVNC_open { + display: flex; + transform: translateY(0); + visibility: visible; + opacity: 1; +} +#noVNC_displays .canvas { + display: flex; + flex-direction: column; + border-radius: 15px; + background: #fdfdfd; + padding: 15px 20px; + box-shadow: 0 0 15px rgba(0,0,0,0.4); + position: relative; +} +#noVNC_displays .canvas-title { + font-size: 14px; + font-weight: bold; + padding: 0 10px; + line-height: 1; + line-height: 1.4; +} +#noVNC_displays .canvas-text { + font-size: 12px; + padding: 0 10px; + opacity: 0.6; + line-height: 1.4; + margin-bottom: 15px; +} +#noVNC_displays .arrange-buttons { + margin-top: 15px; + display: flex; + justify-content: space-between; +} +#noVNC_displays .arrange-buttons button { + border: none; + display: flex; + align-items: center; + padding: 4px 7px; + border-radius: 5px; + cursor: pointer; +} +#noVNC_refreshMonitors { + position: absolute; + top: 20px; + right: 25px; +} +#noVNC_refreshMonitors_icon { + transition: all 0.3s; + transform-origin: center; +} +#noVNC_addMonitor { + background-color: #2196F3; + color: white; +} +#noVNC_addMonitor svg { + margin-right: 5px; +} +#noVNC_displays .canvas canvas { + background: #f7f7f7; + border: 1px solid #ececec; + border-radius: 8px; +} diff --git a/app/ui.js b/app/ui.js index 5cc0e697..85c5af5a 100644 --- a/app/ui.js +++ b/app/ui.js @@ -68,6 +68,9 @@ const UI = { inhibitReconnect: true, reconnectCallback: null, reconnectPassword: null, + monitors: [], + selectedMonitor: null, + refreshRotation: 0, supportsBroadcastChannel: (typeof BroadcastChannel !== "undefined"), @@ -132,7 +135,8 @@ const UI = { UI.addMachineHandlers(); UI.addClipboardHandlers(); UI.addSettingsHandlers(); - UI.addMultiMonitorAddHandler(); + UI.addDisplaysHandler(); + // UI.addMultiMonitorAddHandler(); document.getElementById("noVNC_status") .addEventListener('click', UI.hideStatus); UI.openControlbar(); @@ -574,13 +578,23 @@ const UI = { window.addEventListener('msfullscreenchange', UI.updateFullscreenButton); }, - addMultiMonitorAddHandler() { + addDisplaysHandler() { if (UI.supportsBroadcastChannel) { - UI.showControlInput("noVNC_addmonitor_button"); - UI.addClickHandle('noVNC_addmonitor_button', UI.addSecondaryMonitor); + UI.showControlInput("noVNC_displays_button"); + UI.addClickHandle('noVNC_displays_button', UI.openDisplays); + UI.addClickHandle('noVNC_close_displays', UI.closeDisplays); + UI.addClickHandle('noVNC_addMonitor', UI.addSecondaryMonitor); + UI.addClickHandle('noVNC_refreshMonitors', UI.displaysRefresh); + } }, + /*addMultiMonitorAddHandler() { + if (UI.supportsBroadcastChannel) { + UI.addClickHandle('noVNC_addmonitor_button', UI.addSecondaryMonitor); + } + },*/ + /* ------^------- * /EVENT HANDLERS * ============== @@ -1815,6 +1829,27 @@ const UI = { * /MULTI-MONITOR SUPPORT * ==============*/ + openDisplays() { + document.getElementById('noVNC_displays').classList.add("noVNC_open"); + let screenPlan = UI.rfb.getScreenPlan(); + UI.initMonitors(screenPlan) + UI.displayMonitors() + }, + + closeDisplays() { + document.getElementById('noVNC_displays').classList.remove("noVNC_open"); + }, + + displaysRefresh() { + const rotation = UI.refreshRotation + 180; + let screenPlan = UI.rfb.getScreenPlan(); + document.getElementById('noVNC_refreshMonitors_icon').style.transform = "rotate(" + rotation + "deg)" + UI.refreshRotation = rotation + UI.updateMonitors(screenPlan) + UI.recenter() + UI.draw() + }, + addSecondaryMonitor() { let new_display_path = window.location.pathname.replace(/[^/]*$/, '') let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html`; @@ -1823,6 +1858,261 @@ const UI = { window.open(new_display_url); }, + initMonitors(screenPlan) { + const { scale } = UI.multiMonitorSettings() + let monitors = [] + screenPlan.screens.forEach(screen => { + monitors.push({ + id: screen.screenID, + x: screen.x / scale, + y: screen.y / scale, + w: screen.serverWidth / scale, + h: screen.serverHeight / scale, + scale: 1, + fill: '#eeeeeecc', + isDragging: false + }) + }) + UI.monitors = monitors + }, + + updateMonitors(screenPlan) { + UI.initMonitors(screenPlan) + UI.recenter() + UI.draw() + }, + + multiMonitorSettings() { + const canvas = document.getElementById("noVNC_multiMonitorWidget") + return { + canvas, + ctx: canvas.getContext("2d"), + bb: canvas.getBoundingClientRect(), + scale: 12, + canvasWidth: 560, + canvasHeight: 230, + } + }, + + recenter() { + const monitors = UI.monitors + UI.removeSpaces() + const { startLeft, startTop } = UI.getSizes(monitors) + + for (var i = 0; i < monitors.length; i++) { + var m = monitors[i]; + m.x += startLeft + m.y += startTop + } + }, + + removeSpaces() { + const monitors = UI.monitors + let prev = monitors[0] + if (monitors.length > 1) { + for (var i = 1; i < monitors.length; i++) { + var a = monitors[i]; + let prevStart = prev.x + prev.w + let prevStartTop = prev.y + prev.h + if (a.x > prevStart) { + a.x = prevStart + } + if (a.x < prevStart) { + if (a.y <= prevStartTop) { + a.x = prevStart + } + } + if (a.y > prevStartTop) { + if (a.x <= prevStart) { + a.y = prevStartTop + } + } + prev = monitors[i] + } + } + }, + + rect(ctx, x, y, w, h) { + ctx.beginPath(); + ctx.roundRect(x, y, w, h, 5); + ctx.stroke(); + ctx.closePath(); + ctx.fill(); + }, + + draw() { + const { ctx, canvasWidth, canvasHeight, scale } = UI.multiMonitorSettings() + const monitors = UI.monitors + ctx.clearRect(0, 0, canvasWidth, canvasHeight); + + ctx.rect(0, 0, canvasWidth, canvasHeight); + + for (var i = 0; i < monitors.length; i++) { + var m = monitors[i]; + ctx.fillStyle = m.fill; + ctx.lineWidth = 1; + ctx.lineJoin = "round"; + ctx.strokeStyle = m === UI.selectedMonitor ? "#2196F3" : "#aaa"; + UI.rect(ctx, m.x, m.y, (m.w / m.scale), (m.h / m.scale)); + ctx.font = "13px sans-serif"; + ctx.textAlign = "right"; + ctx.textBaseline = "top"; + ctx.fillStyle = "#000"; + ctx.fillText((i + 1), (m.x + m.w) - 4, m.y + 4); + ctx.font = "200 11px sans-serif"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText(m.w * scale + ' x ' + m.h * scale, m.x + (m.w / 2), m.y + (m.h / 2)); + } + + }, + + getSizes(monitors) { + const { canvasWidth, canvasHeight } = UI.multiMonitorSettings() + let top = monitors[0].y + let left = monitors[0].x + let width = monitors[0].w + let height = monitors[0].h + for (var i = 0; i < monitors.length; i++) { + var m = monitors[i]; + if (m.x < left) { + left = m.x + } + if (m.y < top) { + top = m.y + } + if(m.x + m.w > width) { + width = m.x + m.w + } + if(m.y + m.h > height) { + height = m.y + m.h + } + } + const startLeft = ((canvasWidth - width - left) / 2); + const startTop = ((canvasHeight - height - top) / 2); + + return { top, left, width, height, startLeft, startTop } + }, + + + displayMonitors() { + const { canvas, ctx, bb, canvasWidth, canvasHeight, scale } = UI.multiMonitorSettings() + let offsetX + let offsetY + let dragok = false + let startX; + let startY; + + offsetX = bb.left + offsetY = bb.top + + canvas.addEventListener("mousedown", myDown, false); + canvas.addEventListener("mouseup", myUp, false); + canvas.addEventListener("mousemove", myMove, false); + UI.recenter() + UI.draw() + + function myDown(e) { + let monitors = UI.monitors + e.preventDefault(); + e.stopPropagation(); + let mx = parseInt(e.clientX - offsetX); + let my = parseInt(e.clientY - offsetY); + for (var i = 0; i < monitors.length; i++) { + var mon = monitors[i]; + var monw = mon.w / mon.scale + var monh = mon.h / mon.scale + // Find the closest rect to drag + if (mx > mon.x && mx < mon.x + monw && my > mon.y && my < mon.y + monh) { + dragok = true; + mon.isDragging = true; + UI.selectedMonitor = mon + break // get out of the loop rather than dragging multiple + } + } + startX = mx; + startY = my; + UI.draw() + } + function myUp(e) { + let monitors = UI.monitors + e.preventDefault(); + e.stopPropagation(); + + // clear all the dragging flags + dragok = false; + for (var i = 0; i < monitors.length; i++) { + monitors[i].isDragging = false; + } + const screenplan = setScreenPlan() + UI.recenter() + UI.draw() + } + function myMove(e) { + + if (dragok) { + let monitors = UI.monitors + e.preventDefault(); + e.stopPropagation(); + + // get the current mouse position + var mx = parseInt(e.clientX - offsetX); + var my = parseInt(e.clientY - offsetY); + + // calculate the distance the mouse has moved + // since the last mousemove + var dx = mx - startX; + var dy = my - startY; + + // move each rect that isDragging + // by the distance the mouse has moved + // since the last mousemove + for (var i = 0; i < monitors.length; i++) { + var m = monitors[i]; + if (m.isDragging) { + m.x += dx; + m.y += dy; + if (m.x) { // don't move into another monitor + // if (m.y ) + } + } + } + + // redraw the scene with the new rect positions + UI.draw(); + + // reset the starting mouse position for the next mousemove + startX = mx; + startY = my; + + } + } + + + function setScreenPlan() { + let monitors = UI.monitors + const { top, left, width, height } = UI.getSizes(monitors) + const screens = [] + for (var i = 0; i < monitors.length; i++) { + var a = monitors[i]; + screens.push({ + screenID: a.id, + serverHeight: a.h * scale, + serverWidth: a.w * scale, + x: (a.x - left) * scale, + y: (a.y - top) * scale + }) + } + const screenPlan = { + serverHeight: height * scale, + serverWidth: width * scale, + screens + } + UI.rfb.applyScreenPlan(screenPlan); + } + + }, + /* ------^------- @@ -2478,11 +2768,12 @@ const UI = { let screenPlan = UI.rfb.getScreenPlan(); // Now make adjustments to the screen plan, this is just an example - screenPlan.screens[1].y = 100; + // screenPlan.screens[1].y = 0; // Finally apply the screen plan + UI.rfb.applyScreenPlan(screenPlan); - console.log(screenPlan); + UI.updateMonitors(screenPlan) }, //Helper to add options to dropdown. diff --git a/vnc.html b/vnc.html index 08d581fc..865686d1 100644 --- a/vnc.html +++ b/vnc.html @@ -187,10 +187,11 @@
- - Add Monitor + + Displays
@@ -545,6 +546,23 @@
+ +
+
+
Arrange Displays
+
Drag and drop to arrange displays
+
+ +
+ + +
+
+
+
From f243b54ac9154fdb1efd953520a45ebd3b405aae Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 06:02:33 -0400 Subject: [PATCH 09/41] fix bugs in async frame sync --- core/display.js | 56 ++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/core/display.js b/core/display.js index ab7b4b80..3fc9fa60 100644 --- a/core/display.js +++ b/core/display.js @@ -110,9 +110,6 @@ export default class Display { this.onflush = () => { }; // A flush request has finished - // Use requestAnimationFrame to write to canvas, to match display refresh rate - this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); - if (!this._isPrimaryDisplay) { this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`); this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this)); @@ -482,10 +479,10 @@ export default class Display { */ flush(onflush_message=true) { //force oldest frame to render - this._asyncFrameComplete(0, true); + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); if (onflush_message) - this._flushing = true; + this.onflush(); } /* @@ -501,7 +498,6 @@ export default class Display { */ dispose() { clearInterval(this._frameStatsInterval); - cancelAnimationFrame(this._animationFrameID); this.clear(); } @@ -714,40 +710,45 @@ export default class Display { //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`) switch (rect.type) { case 'copy': - //this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); - this._asyncRenderQPush(rect); + this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); + //this._asyncRenderQPush(rect); break; case 'fill': - this._asyncRenderQPush(rect); - //this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); break; case 'blit': - this._asyncRenderQPush(rect); - //this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'blitQ': - - this._asyncRenderQPush(rect); - //this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); + //this._asyncRenderQPush(rect); + this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); break; case 'img': case '_img': rect.img = new Image(); rect.img.src = rect.src; rect.type = 'img'; - this._asyncRenderQPush(rect); + //this._asyncRenderQPush(rect); + if (!rect.img.complete) { + rect.img.addEventListener('load', (rect) => { + this.drawImage(rect.img, rect.x, rect.y, rect.width, rect.height); + }); + } break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); imageBmpPromise.then(function(rect, img) { - rect.img.complete = true; + //rect.img.complete = true; + this.drawImage(img, rect.x, rect.y, rect.width, rect.height); }).bind(this, rect); - this._asyncRenderQPush(rect); + //this._asyncRenderQPush(rect); break; } break; case 'frameComplete': - this.flip(event.data.frameId, event.data.rectCnt); + //this.flip(event.data.frameId, event.data.rectCnt); break; case 'registered': @@ -847,10 +848,15 @@ export default class Display { } } while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { - if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img' && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { - this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; - this._droppedRects++; + if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') { + if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { + this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; + this._droppedRects++; + } else { + Log.Warn(`Oh snap, an image rect without an image: ${this._asyncFrameQueue[frameIx][2][currentFrameRectIx]}`) + } } + currentFrameRectIx++; } } else { @@ -868,6 +874,8 @@ export default class Display { } this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; this._asyncFrameQueue[frameIx][3] = true; + + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } /* @@ -962,10 +970,6 @@ export default class Display { this._pushAsyncFrame(true); } } - - if (!force) { - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); - } } _processRectScreens(rect) { From 56a2e2dec94ac422d9f73be9b4bfd671ee8bfcf2 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 07:51:50 -0400 Subject: [PATCH 10/41] add syncronous frames for secondary display --- core/base64.js | 2 +- core/display.js | 88 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/core/base64.js b/core/base64.js index e4b6db79..f910ed26 100644 --- a/core/base64.js +++ b/core/base64.js @@ -58,7 +58,7 @@ export default { /* Every four characters is 3 resulting numbers */ const resultLength = (dataLength >> 2) * 3 + Math.floor((dataLength % 4) / 1.5); - const result = new Array(resultLength); + const result = new Uint8Array(resultLength); // Convert one by one. diff --git a/core/display.js b/core/display.js index 3fc9fa60..685192e1 100644 --- a/core/display.js +++ b/core/display.js @@ -12,6 +12,7 @@ import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; import { uuidv4 } from './util/strings.js'; +import base64 from './base64.js'; export default class Display { constructor(target, isPrimaryDisplay) { @@ -31,6 +32,7 @@ export default class Display { this._asyncFrameQueue = []; this._maxAsyncFrameQueue = 3; this._clearAsyncQueue(); + this._syncFrameQueue = []; this._flushing = false; @@ -706,50 +708,24 @@ export default class Display { //overwrite screen locations when received on the secondary display rect.screenLocations = [ rect.screenLocations[event.data.screenLocationIndex] ] rect.screenLocations[0].screenIndex = 0; - let pos = rect.screenLocations[0]; - //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`) switch (rect.type) { - case 'copy': - this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true); - //this._asyncRenderQPush(rect); - break; - case 'fill': - //this._asyncRenderQPush(rect); - this.fillRect(pos.x, pos.y, rect.width, rect.height, rect.color, rect.frame_id, true); - break; - case 'blit': - //this._asyncRenderQPush(rect); - this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); - break; - case 'blitQ': - //this._asyncRenderQPush(rect); - this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true); - break; case 'img': case '_img': rect.img = new Image(); rect.img.src = rect.src; rect.type = 'img'; - //this._asyncRenderQPush(rect); - if (!rect.img.complete) { - rect.img.addEventListener('load', (rect) => { - this.drawImage(rect.img, rect.x, rect.y, rect.width, rect.height); - }); - } break; case 'transparent': let imageBmpPromise = createImageBitmap(rect.arr); imageBmpPromise.then(function(rect, img) { - //rect.img.complete = true; - this.drawImage(img, rect.x, rect.y, rect.width, rect.height); + rect.img.complete = true; }).bind(this, rect); - //this._asyncRenderQPush(rect); break; } + this._syncFrameQueue.push(rect); break; case 'frameComplete': - //this.flip(event.data.frameId, event.data.rectCnt); - + window.requestAnimationFrame( () => { this._pushSyncRects(); }); break; case 'registered': if (!this._isPrimaryDisplay) { @@ -761,6 +737,59 @@ export default class Display { } } + _pushSyncRects() { + whileLoop: + while (this._syncFrameQueue.length > 0) { + const a = this._syncFrameQueue[0]; + const pos = a.screenLocations[0]; + switch (a.type) { + case 'copy': + this.copyImage(pos.oldX, pos.oldY, pos.x, pos.y, a.width, a.height, a.frame_id, true); + break; + case 'fill': + this.fillRect(pos.x, pos.y, a.width, a.height, a.color, a.frame_id, true); + break; + case 'blit': + this.blitImage(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'blitQ': + this.blitQoi(pos.x, pos.y, a.width, a.height, a.data, 0, a.frame_id, true); + break; + case 'img': + if (a.img.complete) { + this.drawImage(a.img, pos.x, pos.y, a.width, a.height); + } else { + if (this._syncFrameQueue.length > 1000) { + this._syncFrameQueue.shift(); + this._droppedRects++; + } else { + break whileLoop; + } + } + break; + case 'transparent': + if (a.img.complete) { + this.drawImage(a.img, pos.x, pos.y, a.width, a.height); + } else { + if (this._syncFrameQueue.length > 1000) { + this._syncFrameQueue.shift(); + this._droppedRects++; + } else { + break whileLoop; + } + } + break; + default: + Log.Warn(`Unknown rect type: ${rect}`); + } + this._syncFrameQueue.shift(); + } + + if (this._syncFrameQueue.length > 0) { + window.requestAnimationFrame( () => { this._pushSyncRects(); }); + } + } + /* Process incoming rects into a frame buffer, assume rects are out of order due to either UDP or parallel processing of decoding */ @@ -924,6 +953,7 @@ export default class Display { if (a.img) { a.img = null; } + if (a.type !== 'flip') { secondaryScreenRects++; this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: a, screenLocationIndex: sI }); From 70faacff85bfc2c0c58d77a7cc29ac0abd0751ec Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 09:19:00 -0400 Subject: [PATCH 11/41] smashed a frame dropping bug --- core/display.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/display.js b/core/display.js index 685192e1..d19e05d6 100644 --- a/core/display.js +++ b/core/display.js @@ -115,6 +115,8 @@ export default class Display { if (!this._isPrimaryDisplay) { this._screens[0].channel = new BroadcastChannel(`screen_${this._screenID}_channel`); this._screens[0].channel.addEventListener('message', this._handleSecondaryDisplayMessage.bind(this)); + } else { + this._animationFrameID = window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } Log.Debug("<< Display.constructor"); @@ -481,7 +483,8 @@ export default class Display { */ flush(onflush_message=true) { //force oldest frame to render - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + //window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + this._asyncFrameComplete(0, true); if (onflush_message) this.onflush(); @@ -826,7 +829,7 @@ export default class Display { } } - if (this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { + if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { //frame is complete this._asyncFrameComplete(frameIx); } @@ -904,7 +907,7 @@ export default class Display { this._asyncFrameQueue[frameIx][4] = currentFrameRectIx; this._asyncFrameQueue[frameIx][3] = true; - window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + //window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); } /* @@ -1000,6 +1003,10 @@ export default class Display { this._pushAsyncFrame(true); } } + + if (!force) { + window.requestAnimationFrame( () => { this._pushAsyncFrame(); }); + } } _processRectScreens(rect) { From 0f4979bcc580574cf6711f1e865f8aa33b117acd Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Thu, 5 Oct 2023 13:30:37 -0400 Subject: [PATCH 12/41] several fixes --- app/ui_screen.js | 8 ++++++ core/display.js | 45 +++++++++++++++++++++++---------- core/rfb.js | 66 +++++++++++++----------------------------------- 3 files changed, 57 insertions(+), 62 deletions(-) diff --git a/app/ui_screen.js b/app/ui_screen.js index 5e193979..cc2bb792 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -114,6 +114,8 @@ const UI = { parent.postMessage({ action: 'connection_state', value: state}, '*' ); } + let connect_el = document.getElementById('noVNC_connect_dlg'); + switch (state) { case 'init': break; @@ -123,6 +125,9 @@ const UI = { break; case 'connected': document.documentElement.classList.add("noVNC_connected"); + if (!connect_el.classList.contains("noVNC_hidden")) { + connect_el.classList.add('noVNC_hidden'); + } break; case 'disconnecting': transitionElem.textContent = _("Disconnecting..."); @@ -130,6 +135,9 @@ const UI = { break; case 'disconnected': document.documentElement.classList.add("noVNC_disconnected"); + if (connect_el.classList.contains("noVNC_hidden")) { + connect_el.classList.remove('noVNC_hidden'); + } break; case 'reconnecting': transitionElem.textContent = _("Reconnecting..."); diff --git a/core/display.js b/core/display.js index d19e05d6..4046cae3 100644 --- a/core/display.js +++ b/core/display.js @@ -12,7 +12,6 @@ import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; import { uuidv4 } from './util/strings.js'; -import base64 from './base64.js'; export default class Display { constructor(target, isPrimaryDisplay) { @@ -203,8 +202,10 @@ export default class Display { //recalculate primary display container size this._screens[0].containerHeight = this._target.parentNode.offsetHeight; this._screens[0].containerWidth = this._target.parentNode.offsetWidth; - this._screens[0].width = this._target.width; - this._screens[0].height = this._target.height; + this._screens[0].width = this._target.parentNode.offsetWidth; + this._screens[0].height = this._target.parentNode.offsetHeight; + //this._screens[0].width = this._target.width; + //this._screens[0].height = this._target.height; //calculate server-side resolution of each screen for (let i=0; i 0) { + //existing screen, update + const screen = this._screens[screenIdx]; + screen.width = width; + screen.height = height; + screen.containerHeight = containerHeight; + screen.containerWidth = containerWidth; + screen.pixelRatio = pixelRatio; + + } else { + //New Screen, add to far right until user repositions it let x = 0; for (let i = 0; i < this._screens.length; i++) { x = Math.max(x, this._screens[i].x + this._screens[i].width); @@ -296,10 +320,7 @@ export default class Display { this._screens.push(new_screen); new_screen.channel.postMessage({ eventType: "registered", screenIndex: new_screen.screenIndex }); - } else { - throw new Error("Cannot add a screen to a secondary display.") } - } removeScreen(screenID) { @@ -1017,11 +1038,9 @@ export default class Display { rect.inSecondary = false; for (let i=0; i < this._screens.length; i++) { let screen = this._screens[i]; - if ( - ((rect.x >= screen.x && rect.x < screen.x + screen.width) && - (rect.y >= screen.y && rect.y < screen.y + screen.height)) || - ((rect.x+rect.width >= screen.x && rect.x+rect.width < screen.x + screen.width) && - (rect.y+rect.height >= screen.y && rect.y+rect.height < screen.y + screen.height)) + + if ( + !((rect.x > screen.x2 || screen.x > (rect.x + rect.width)) && (rect.y > screen.y2 || screen.y > (rect.y + rect.height))) ) { let screenPosition = { x: 0 - (screen.x - rect.x), //rect.x - screen.x, diff --git a/core/rfb.js b/core/rfb.js index 42d6e4b8..95428358 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -828,6 +828,7 @@ export default class RFB extends EventTargetMixin { this._sock.off('error'); this._sock.off('message'); this._sock.off('open'); + this._proxyRFBMessage('disconnect'); } else { this._updateConnectionState('disconnecting'); this._unregisterSecondaryDisplay(); @@ -1467,60 +1468,25 @@ export default class RFB extends EventTargetMixin { clearTimeout(this._resizeTimeout); this._resizeTimeout = null; - if (!this._resizeSession || this._viewOnly || - !this._supportsSetDesktopSize) { - return; - } - const size = this._screenSize(); - RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); + if (this._isPrimaryDisplay) { + if (!this._resizeSession || this._viewOnly || + !this._supportsSetDesktopSize) { + return; + } + const size = this._screenSize(); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); - Log.Debug('Requested new desktop size: ' + + Log.Debug('Requested new desktop size: ' + size.serverWidth + 'x' + size.serverHeight); + } else if (this._display.screenIndex > 0) { + //re-register the secondary display with new resolution + this._registerSecondaryDisplay(); + } } // Gets the the size of the available screen _screenSize (limited) { return this._display.getScreenSize(this.videoQuality, this.forcedResolutionX, this.forcedResolutionY, this._hiDpi, limited); - - if (limited === undefined) { - limited = true; - } - var x = this.forcedResolutionX || this._screen.offsetWidth; - var y = this.forcedResolutionY || this._screen.offsetHeight; - var scale = 0; // 0=auto - try { - if (x > 1280 && limited && this.videoQuality == 1) { - var ratio = y / x; - Log.Debug(ratio); - x = 1280; - y = x * ratio; - } - else if (limited && this.videoQuality == 0){ - x = 1280; - y = 720; - } else if (this._hiDpi == true) { - x = x * window.devicePixelRatio; - y = y * window.devicePixelRatio; - scale = 1 / window.devicePixelRatio; - } else if (this._display.antiAliasing === 0 && window.devicePixelRatio > 1 && x < 1000 && x > 0) { - // small device with high resolution, browser is essentially zooming greater than 200% - Log.Info('Device Pixel ratio: ' + window.devicePixelRatio + ' Reported Resolution: ' + x + 'x' + y); - let targetDevicePixelRatio = 1.5; - if (window.devicePixelRatio > 2) { targetDevicePixelRatio = 2; } - let scaledWidth = (x * window.devicePixelRatio) * (1 / targetDevicePixelRatio); - let scaleRatio = scaledWidth / x; - x = x * scaleRatio; - y = y * scaleRatio; - scale = 1 / scaleRatio; - Log.Info('Small device with hDPI screen detected, auto scaling at ' + scaleRatio + ' to ' + x + 'x' + y); - } - } catch (err) { - Log.Debug(err); - } - - return { w: x, - h: y, - scale: scale }; } _fixScrollbars() { @@ -1611,7 +1577,7 @@ export default class RFB extends EventTargetMixin { this._disconnTimer = null; // make sure we don't get a double event - if (this._rfbConnectionState !== 'proxied') { + if (this._isPrimaryDisplay) { this._sock.off('close'); } } @@ -1714,7 +1680,6 @@ export default class RFB extends EventTargetMixin { this._mouseLastScreenIndex = event.data.screenIndex; event.data.args[0] = coords[0]; event.data.args[1] = coords[1]; - console.log(`screenIndex ${event.data.screenIndex}, x: ${coords[0]}, y: ${coords[1]}`); RFB.messages.pointerEvent(this._sock, ...event.data.args); break; case 'keyEvent': @@ -1737,6 +1702,8 @@ export default class RFB extends EventTargetMixin { this._updateCursor(...event.data.args); } break; + case 'disconnect': + this.disconnect(); } } @@ -4312,6 +4279,7 @@ RFB.messages = { }, setDesktopSize(sock, size, flags) { + console.log(size); const buff = sock._sQ; const offset = sock._sQlen; From 954428d6c83ee6c83ca86769efb15c312be3402d Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Wed, 11 Oct 2023 08:31:08 +0000 Subject: [PATCH 13/41] Fix spacing and fractional units --- app/ui.js | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/app/ui.js b/app/ui.js index 85c5af5a..712f63e9 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1690,6 +1690,14 @@ const UI = { UI.toggleIMEMode(); } break; + case 'open_displays_mode': + if (UI.rfb) { + UI.openDisplays() + } + break; + case 'close_displays_mode': + UI.closeDisplays() + break; case 'enable_webrtc': if (!UI.getSetting('enable_webrtc')) { UI.forceSetting('enable_webrtc', true, false); @@ -2044,8 +2052,8 @@ const UI = { for (var i = 0; i < monitors.length; i++) { monitors[i].isDragging = false; } - const screenplan = setScreenPlan() UI.recenter() + const screenplan = setScreenPlan() UI.draw() } function myMove(e) { @@ -2097,15 +2105,15 @@ const UI = { var a = monitors[i]; screens.push({ screenID: a.id, - serverHeight: a.h * scale, - serverWidth: a.w * scale, - x: (a.x - left) * scale, - y: (a.y - top) * scale + serverHeight: Math.floor(a.h * scale), + serverWidth: Math.floor(a.w * scale), + x: Math.floor((a.x - left) * scale), + y: Math.floor((a.y - top) * scale) }) } const screenPlan = { - serverHeight: height * scale, - serverWidth: width * scale, + serverHeight: Math.floor(height * scale), + serverWidth: Math.floor(width * scale), screens } UI.rfb.applyScreenPlan(screenPlan); From 19e8e924c24e3eb2e2f7d725730189bbf38e9506 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 11 Oct 2023 06:27:14 -0400 Subject: [PATCH 14/41] Fix native resolution on hdpi screens --- core/display.js | 18 +++++++++--------- core/rfb.js | 16 +++++++++++----- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/core/display.js b/core/display.js index 4046cae3..833128f8 100644 --- a/core/display.js +++ b/core/display.js @@ -245,13 +245,13 @@ export default class Display { this._screens[i].serverWidth = width; this._screens[i].serverHeight = height; this._screens[i].scale = scale; - this._screens[i].x2 = this._screens[i].x + this._screens[i].width; - this._screens[i].y2 = this._screens[i].y + this._screens[i].height; + this._screens[i].x2 = this._screens[i].x + this._screens[i].serverWidth; + this._screens[i].y2 = this._screens[i].y + this._screens[i].serverHeight; } for (let i = 0; i < this._screens.length; i++) { - data.serverWidth = Math.max(data.serverWidth, this._screens[i].x + this._screens[i].width); - data.serverHeight = Math.max(data.serverHeight, this._screens[i].y + this._screens[i].height); + data.serverWidth = Math.max(data.serverWidth, this._screens[i].x + this._screens[i].serverWidth); + data.serverHeight = Math.max(data.serverHeight, this._screens[i].y + this._screens[i].serverHeight); } data.screens = this._screens; @@ -296,7 +296,7 @@ export default class Display { //New Screen, add to far right until user repositions it let x = 0; for (let i = 0; i < this._screens.length; i++) { - x = Math.max(x, this._screens[i].x + this._screens[i].width); + x = Math.max(x, this._screens[i].x + this._screens[i].serverWidth); } var new_screen = { @@ -1038,7 +1038,7 @@ export default class Display { rect.inSecondary = false; for (let i=0; i < this._screens.length; i++) { let screen = this._screens[i]; - + if ( !((rect.x > screen.x2 || screen.x > (rect.x + rect.width)) && (rect.y > screen.y2 || screen.y > (rect.y + rect.height))) ) { @@ -1071,8 +1071,8 @@ export default class Display { // style width to a number, the canvas is cleared. // However, if you set the style width to a string // ('NNNpx'), the canvas is scaled without clearing. - const width = factor * vp.width + 'px'; - const height = factor * vp.height + 'px'; + const width = factor * vp.serverWidth + 'px'; + const height = factor * vp.serverHeight + 'px'; if ((this._target.style.width !== width) || (this._target.style.height !== height)) { @@ -1080,7 +1080,7 @@ export default class Display { this._target.style.height = height; } - Log.Info('Pixel Ratio: ' + window.devicePixelRatio + ', VNC Scale: ' + factor + 'VNC Res: ' + vp.width + 'x' + vp.height); + Log.Info('Pixel Ratio: ' + window.devicePixelRatio + ', VNC Scale: ' + factor + 'VNC Res: ' + vp.serverWidth + 'x' + vp.serverHeight); var pixR = Math.abs(Math.ceil(window.devicePixelRatio)); var isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; diff --git a/core/rfb.js b/core/rfb.js index 95428358..b659ee50 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -728,6 +728,10 @@ export default class RFB extends EventTargetMixin { if (value !== this._hiDpi) { this._hiDpi = value; this._requestRemoteResize(); + if (this._display.screens.length > 1) { + //force secondary displays to re-register and thus apply new hdpi setting + this._proxyRFBMessage('forceResize', [ value ]); + } } } @@ -741,7 +745,7 @@ export default class RFB extends EventTargetMixin { applyScreenPlan(screenPlan) { if (this._isPrimaryDisplay) { - let fullPlan = this._display.getScreenSize(); + let fullPlan = this._screenSize(); //check plan for validity let minX = Number.MAX_SAFE_INTEGER, minY = Number.MAX_SAFE_INTEGER; @@ -773,7 +777,7 @@ export default class RFB extends EventTargetMixin { } getScreenPlan() { - let fullPlan = this._display.getScreenSize(); + let fullPlan = this._screenSize(); let sanitizedPlan = { screens: [], serverWidth: fullPlan.serverWidth, @@ -1457,7 +1461,7 @@ export default class RFB extends EventTargetMixin { this._display.scale = 1.0; } else { const size = this._screenSize(false); - this._display.autoscale(size.screens[0].containerWidth, size.screens[0].containerHeight, size.screens[0].scale); + this._display.autoscale(size.screens[0].serverWidth, size.screens[0].serverHeight, size.screens[0].scale); } this._fixScrollbars(); } @@ -1691,8 +1695,6 @@ export default class RFB extends EventTargetMixin { // The following are primary to secondary messages that should be ignored on the primary case 'updateCursor': break; - default: - Log.Warn(`Unhandled message type (${event.data.eventType}) from control channel.`); } } else { // Primary to secondary screen message @@ -1704,6 +1706,10 @@ export default class RFB extends EventTargetMixin { break; case 'disconnect': this.disconnect(); + case 'forceResize': + this._hiDpi = event.data.args[0]; + this._updateScale(); + this._requestRemoteResize(); } } From 3cc2f0caf32c3c7f67e10f95b57656cb947ae159 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 11 Oct 2023 06:34:17 -0400 Subject: [PATCH 15/41] add pixelratio to screen plan --- core/rfb.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/rfb.js b/core/rfb.js index b659ee50..25872245 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -791,7 +791,8 @@ export default class RFB extends EventTargetMixin { serverWidth: fullPlan.screens[i].serverWidth, serverHeight: fullPlan.screens[i].serverHeight, x: fullPlan.screens[i].x, - y: fullPlan.screens[i].y + y: fullPlan.screens[i].y, + pixelRatio: fullPlan.screens[i].pixelRatio } ) } From 05ddc913820e69fad0dc878040302a8ea126c853 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 11 Oct 2023 09:27:04 -0400 Subject: [PATCH 16/41] handle window moved to diffent monitor --- core/display.js | 1 + core/rfb.js | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/display.js b/core/display.js index 833128f8..bcf36177 100644 --- a/core/display.js +++ b/core/display.js @@ -204,6 +204,7 @@ export default class Display { this._screens[0].containerWidth = this._target.parentNode.offsetWidth; this._screens[0].width = this._target.parentNode.offsetWidth; this._screens[0].height = this._target.parentNode.offsetHeight; + this._screens[0].pixelRatio = window.devicePixelRatio; //this._screens[0].width = this._target.width; //this._screens[0].height = this._target.height; diff --git a/core/rfb.js b/core/rfb.js index 25872245..cfaa9b98 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -239,6 +239,7 @@ export default class RFB extends EventTargetMixin { handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), handleFocusChange: this._handleFocusChange.bind(this), + handleMouseOut: this._handleMouseOut.bind(this), }; // main setup @@ -1163,6 +1164,9 @@ export default class RFB extends EventTargetMixin { window.addEventListener("focus", this._eventHandlers.handleFocusChange); window.addEventListener("blur", this._eventHandlers.handleFocusChange); + //User cursor moves outside of the window + window.addEventListener("mouseover", this._eventHandlers.handleMouseOut); + // In order for the keyboard to not occlude the input being edited // we move the hidden input we use for triggering the keyboard to the last click // position which should trigger a page being moved down enough @@ -1757,7 +1761,6 @@ export default class RFB extends EventTargetMixin { } _handleSecondaryDisplayMessage(event) { - console.log("Message Received: " + event); if (this._isPrimaryDisplay) { } @@ -1796,6 +1799,18 @@ export default class RFB extends EventTargetMixin { this.sendKey(keysym, code, down); } + _handleMouseOut(ev) { + if (ev.toElement !== null && ev.relatedTarget === null) { + //mouse was outside of the window and just came in, this is our chance to do things + + //Ensure the window was not moved to a different screen with a different pixel ratio + if (this._display.screens[0].pixelRatio !== window.devicePixelRatio) { + Log.Debug("Window moved to another screen with different pixel ratio, sending resize request."); + this._requestRemoteResize(); + } + } + } + _handleMouse(ev) { /* * We don't check connection status or viewOnly here as the @@ -4286,7 +4301,6 @@ RFB.messages = { }, setDesktopSize(sock, size, flags) { - console.log(size); const buff = sock._sQ; const offset = sock._sQlen; From 02745eee0aed57646e8da1b25492cfc0a6f1cea1 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Thu, 12 Oct 2023 12:02:44 +0100 Subject: [PATCH 17/41] Add native resolution to displays --- app/styles/base.css | 8 ++++++-- app/ui.js | 49 +++++++++++++++++++++++---------------------- vnc.html | 20 +++++++++--------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/app/styles/base.css b/app/styles/base.css index c45ca2ca..e97c2905 100644 --- a/app/styles/base.css +++ b/app/styles/base.css @@ -1281,13 +1281,16 @@ a:visited { display: flex; justify-content: space-between; } -#noVNC_displays .arrange-buttons button { +#noVNC_displays .arrange-buttons button, #noVNC_displays .arrange-buttons .button { border: none; display: flex; align-items: center; padding: 4px 7px; border-radius: 5px; cursor: pointer; + background-color: #f1f1f1; + font-size: 13px; + border: 1px solid #e5e5e5; } #noVNC_refreshMonitors { position: absolute; @@ -1299,8 +1302,9 @@ a:visited { transform-origin: center; } #noVNC_addMonitor { - background-color: #2196F3; + background-color: #2196F3!important; color: white; + border: none!important; } #noVNC_addMonitor svg { margin-right: 5px; diff --git a/app/ui.js b/app/ui.js index 712f63e9..3a1270af 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1912,6 +1912,7 @@ const UI = { m.x += startLeft m.y += startTop } + UI.setScreenPlan() }, removeSpaces() { @@ -2002,6 +2003,30 @@ const UI = { return { top, left, width, height, startLeft, startTop } }, + setScreenPlan() { + let monitors = UI.monitors + const { scale } = UI.multiMonitorSettings() + const { top, left, width, height } = UI.getSizes(monitors) + const screens = [] + for (var i = 0; i < monitors.length; i++) { + var a = monitors[i]; + screens.push({ + screenID: a.id, + serverHeight: Math.floor(a.h * scale), + serverWidth: Math.floor(a.w * scale), + x: Math.floor((a.x - left) * scale), + y: Math.floor((a.y - top) * scale) + }) + } + const screenPlan = { + serverHeight: Math.floor(height * scale), + serverWidth: Math.floor(width * scale), + screens + } + UI.rfb.applyScreenPlan(screenPlan); + }, + + displayMonitors() { const { canvas, ctx, bb, canvasWidth, canvasHeight, scale } = UI.multiMonitorSettings() @@ -2053,7 +2078,6 @@ const UI = { monitors[i].isDragging = false; } UI.recenter() - const screenplan = setScreenPlan() UI.draw() } function myMove(e) { @@ -2096,29 +2120,6 @@ const UI = { } } - - function setScreenPlan() { - let monitors = UI.monitors - const { top, left, width, height } = UI.getSizes(monitors) - const screens = [] - for (var i = 0; i < monitors.length; i++) { - var a = monitors[i]; - screens.push({ - screenID: a.id, - serverHeight: Math.floor(a.h * scale), - serverWidth: Math.floor(a.w * scale), - x: Math.floor((a.x - left) * scale), - y: Math.floor((a.y - top) * scale) - }) - } - const screenPlan = { - serverHeight: Math.floor(height * scale), - serverWidth: Math.floor(width * scale), - screens - } - UI.rfb.applyScreenPlan(screenPlan); - } - }, diff --git a/vnc.html b/vnc.html index 865686d1..1ccc0354 100644 --- a/vnc.html +++ b/vnc.html @@ -299,12 +299,6 @@ Toggle Control Panel via Keystrokes -
  • - -
  • + Native Resolution + +
    • From c7c515796a6322baf90f926ab679ad8f3617af5c Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Thu, 12 Oct 2023 13:59:56 +0100 Subject: [PATCH 18/41] Show native resolution option Remove display when deleted --- app/styles/base.css | 7 +++++++ app/ui.js | 12 +++++++++++- core/rfb.js | 2 ++ vnc.html | 4 ++-- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/app/styles/base.css b/app/styles/base.css index e97c2905..b43fb54c 100644 --- a/app/styles/base.css +++ b/app/styles/base.css @@ -1291,6 +1291,13 @@ a:visited { background-color: #f1f1f1; font-size: 13px; border: 1px solid #e5e5e5; + line-height: 1; +} +#noVNC_setting_enable_hidpi_option { + display: none!important; +} +#noVNC_setting_enable_hidpi_option.show { + display: flex!important; } #noVNC_refreshMonitors { position: absolute; diff --git a/app/ui.js b/app/ui.js index 3a1270af..50aad392 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1869,18 +1869,28 @@ const UI = { initMonitors(screenPlan) { const { scale } = UI.multiMonitorSettings() let monitors = [] + let showNativeResolution = false screenPlan.screens.forEach(screen => { + if (parseFloat(screen.pixelRatio) != 1) { + showNativeResolution = true + } monitors.push({ id: screen.screenID, x: screen.x / scale, y: screen.y / scale, w: screen.serverWidth / scale, h: screen.serverHeight / scale, + pixelRatio: screen.pixelRatio, scale: 1, fill: '#eeeeeecc', isDragging: false }) }) + if (showNativeResolution) { + document.getElementById('noVNC_setting_enable_hidpi_option').classList.add("show"); + } else { + document.getElementById('noVNC_setting_enable_hidpi_option').classList.remove("show"); + } UI.monitors = monitors }, @@ -2775,7 +2785,7 @@ const UI = { // Get the current screen plan // When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top let screenPlan = UI.rfb.getScreenPlan(); - + console.log(e) // Now make adjustments to the screen plan, this is just an example // screenPlan.screens[1].y = 0; diff --git a/core/rfb.js b/core/rfb.js index cfaa9b98..69bd0964 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1491,6 +1491,7 @@ export default class RFB extends EventTargetMixin { //re-register the secondary display with new resolution this._registerSecondaryDisplay(); } + this.dispatchEvent(new CustomEvent("screenregistered", {})); } // Gets the the size of the available screen @@ -1677,6 +1678,7 @@ export default class RFB extends EventTargetMixin { break; case 'unregister': if (this._display.removeScreen(event.data.screenID)) { + this.dispatchEvent(new CustomEvent("screenregistered", {})); Log.Info(`Secondary monitor (${event.data.screenID}) has been removed.`); const size = this._screenSize(); RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); diff --git a/vnc.html b/vnc.html index 1ccc0354..024f4471 100644 --- a/vnc.html +++ b/vnc.html @@ -553,8 +553,8 @@ Add Monitor -