From aef462ce62a9f6f169705cfbc42d4e905387e7b4 Mon Sep 17 00:00:00 2001 From: mattmcclaskey Date: Wed, 20 Sep 2023 15:18:08 -0400 Subject: [PATCH] refactor to support any number of displays in any orientation --- app/ui.js | 14 +++ app/ui_screen.js | 15 +--- core/display.js | 215 +++++++++++++---------------------------------- core/rfb.js | 97 ++++++++++++++++----- 4 files changed, 149 insertions(+), 192 deletions(-) diff --git a/app/ui.js b/app/ui.js index f866bd3c..5cc0e697 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1388,6 +1388,7 @@ const UI = { UI.rfb.addEventListener("desktopname", UI.updateDesktopName); UI.rfb.addEventListener("inputlock", UI.inputLockChanged); UI.rfb.addEventListener("inputlockerror", UI.inputLockError); + UI.rfb.addEventListener("screenregistered", UI.screenRegistered); UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts'); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; @@ -2471,6 +2472,19 @@ const UI = { } }, + screenRegistered(e) { + // Get the current screen plan + // When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top + let screenPlan = UI.rfb.getScreenPlan(); + + // Now make adjustments to the screen plan, this is just an example + screenPlan.screens[1].y = 100; + + // Finally apply the screen plan + UI.rfb.applyScreenPlan(screenPlan); + console.log(screenPlan); + }, + //Helper to add options to dropdown. addOption(selectbox, text, value) { const optn = document.createElement("OPTION"); diff --git a/app/ui_screen.js b/app/ui_screen.js index d66a9416..5e193979 100644 --- a/app/ui_screen.js +++ b/app/ui_screen.js @@ -92,19 +92,8 @@ const UI = { UI.rfb.enableQOI = true; } - // attach secondary display with relative position, relative x, and relative y - // relativePosition: - // 0: primary display is to left - // 1: primary display is up top - // 2: primary display is to right - // 3: primary display is down below - // relativePositionX: - // non-zero number only allowed if relativePosition is 1 or 3 - // How many pixels on the X axis is the secondary screens starting position from the primary displays - // relativePositionY: - // non-zero number only allowed if relativePosition is 0 or 2 - // How many pixels on the Y axis is the secondary screens starting position from the primary displays - UI.rfb.attachSecondaryDisplay(3, 0, 0); + //attach this secondary display to the primary display + UI.rfb.attachSecondaryDisplay(); if (supportsBinaryClipboard()) { // explicitly request permission to the clipboard diff --git a/core/display.js b/core/display.js index 6b690b86..ab7b4b80 100644 --- a/core/display.js +++ b/core/display.js @@ -11,7 +11,7 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; import { toSigned32bit } from './util/int.js'; import { isWindows } from './util/browser.js'; -import { uuidv4 } from './util/strings.js' +import { uuidv4 } from './util/strings.js'; export default class Display { constructor(target, isPrimaryDisplay) { @@ -81,6 +81,7 @@ export default class Display { // ===== PROPERTIES ===== + this._maxScreens = 4; this._scale = 1.0; this._clipViewport = false; this._antiAliasing = 0; @@ -96,9 +97,9 @@ export default class Display { serverHeight: 0, //calculated x: 0, y: 0, - relativePosition: 0, - relativePositionX: 0, - relativePositionY: 0, + relativePosition: 0, //left, right, up, down relative to primary display + relativePositionX: 0, //offset relative to primary monitor, always 0 for primary + relativePositionY: 0, //offset relative to primary monitor, always 0 for primary pixelRatio: window.devicePixelRatio, containerHeight: this._target.parentNode.offsetHeight, containerWidth: this._target.parentNode.offsetWidth, @@ -122,32 +123,16 @@ export default class Display { // ===== PROPERTIES ===== - get relativePosition() { return this._screens[0].relativePosition; } - set relativePosition(value) { - if (!this._isPrimaryDisplay && value >= 0 && value < 4) { - this._screens[0].relativePosition = value; - //reset relative X and Y - this._screens[0].relativePositionX = 0; - this._screens[0].relativePositionY = 0; - } - } - - get relativePositionX() { return this._screens[0].relativePositionX; } - set relativePositionX(value) { - if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 1 || this._screens[0].relativePosition == 3)) { - this._screens[0].relativePositionX = value; - } - } - - get relativePositionY() { return this._screens[0].relativePositionY; } - set relativePositionY(value) { - if (!this._isPrimaryDisplay && (this._screens[0].relativePosition == 0 || this._screens[0].relativePosition == 2)) { - this._screens[0].relativePositionY = value; - } - } - get screens() { return this._screens; } get screenId() { return this._screenID; } + get screenIndex() { + // A secondary screen should not have a screen index of 0, but it will be 0 until registration is complete + // returning a -1 lets the caller know the screen has not been registered yet + if (!this._isPrimaryDisplay && this._screens[0].screenIndex == 0) { + return -1; + } + return this._screens[0].screenIndex; + } get antiAliasing() { return this._antiAliasing; } set antiAliasing(value) { @@ -189,62 +174,21 @@ export default class Display { // ===== PUBLIC METHODS ===== /* - Returns coordinates that are client relative when multiple monitors are in use - Returns an array with the following - 0 - screen index - 1 - screenId - 2 - x - 3 - y + Returns the screen index given serverside relative coordinates */ - getClientRelativeCoordinates(x, y) { - if (this._screens.length == 1) { - return [ 0, this._screenID, x, y ]; - } - //TODO: The following logic will only support two monitors placed horizontally - let screenOrientation = this._screens[1].relativePosition; - let screenIdx = 0; - let screenId = this._screens[0].screenID; - if (screenOrientation == 0) { - if (x >= this._screens[1].x) { - x -= this._screens[1].x; - screenIdx = 1; - screenId = this._screens[1].screenID; - } - } else if (screenOrientation == 2) { - if (x >= this._screens[0].x) { - x -= this._screens[0].x; - } - } - return [ screenIdx, screenId, x, y ]; + getScreenIndexByServerCoords(x, y) { + } /* Returns coordinates that are server relative when multiple monitors are in use */ - getServerRelativeCoordinates(screenId, x, y) { - // If this is the primary screen and only one screen, lets keep it simple - if (this._isPrimaryDisplay && this._screens.length == 1) { - return [x, y]; + getServerRelativeCoordinates(screenIndex, x, y) { + if (screenIndex >= 0 && screenIndex < this._screens.length) { + x += this._screens[screenIndex].x; + y += this._screens[screenIndex].y; } - // Find the screen index by ID - let screenIdx = -1; - for (let i=0; i 1) { - - const secondary_screen = this._screens[1]; - secondary_screen.x = 0; - secondary_screen.y = 0; - - switch (this._screens[1].relativePosition) { - case 0: - //primary screen is to left - total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; - total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); - secondary_screen.x = primary_screen.serverWidth; - - if (secondary_screen.relativePositionY >= 0) { - secondary_screen.y = secondary_screen.relativePositionY; - } else { - primary_screen.y = Math.abs(secondary_screen.relativePositionY); - } - - break; - case 1: - //primary screen is up above - total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); - total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; - secondary_screen.y = primary_screen.serverHeight; - - if (secondary_screen.relativePositionX >= 0) { - secondary_screen.x = secondary_screen.relativePositionX; - } else { - primary_screen.x = Math.abs(secondary_screen.relativePositionX); - } - break; - case 2: - //primary screen is to right - total_server_width = secondary_screen.serverWidth + primary_screen.serverWidth; - total_server_height = Math.max(primary_screen.serverHeight, secondary_screen.serverHeight) + Math.abs(secondary_screen.relativePositionY); - primary_screen.x = secondary_screen.serverWidth; - - if (secondary_screen.relativePositionY >= 0) { - secondary_screen.y = secondary_screen.relativePositionY; - } else { - primary_screen.y = Math.abs(secondary_screen.relativePositionY); - } - break; - case 3: - //primary screen is down below - total_server_width = Math.max(primary_screen.serverWidth, secondary_screen.serverWidth) + Math.abs(secondary_screen.relativePositionX); - total_server_height = secondary_screen.serverHeight + primary_screen.serverHeight; - primary_screen.y = secondary_screen.serverHeight; - - if (secondary_screen.relativePositionX >= 0) { - secondary_screen.x = secondary_screen.relativePositionX; - } else { - primary_screen.x = Math.abs(secondary_screen.relativePositionX); - } - break; - default: - //TODO: It would not be hard to support vertically stacked monitors - throw new Error("Unsupported screen orientation."); - } + for (let i = 0; i < this._screens.length; i++) { + data.serverWidth = Math.max(data.serverWidth, this._screens[i].x + this._screens[i].width); + data.serverHeight = Math.max(data.serverHeight, this._screens[i].y + this._screens[i].height); } data.screens = this._screens; - data.serverWidth = total_server_width; - data.serverHeight = total_server_height; return data; } - addScreen(screenID, width, height, relativePosition, relativePositionX, relativePositionY, pixelRatio, containerHeight, containerWidth) { + applyScreenPlan(screenPlan) { + for (let i = 0; i < screenPlan.screens.length; i++) { + for (let z = 0; z < this._screens.length; z++) { + if (screenPlan.screens[i].screenID === this._screens[z].screenID) { + this._screens[z].x = screenPlan.screens[i].x; + this._screens[z].y = screenPlan.screens[i].y; + } + } + } + } + + addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth) { if (this._isPrimaryDisplay) { - //currently only support one secondary screen - if (this._screens.length > 1) { - this._screens[1].channel.close(); - this._screens.pop() + //for now, place new screen to the far right, until the user repositions it + let x = 0; + for (let i = 0; i < this._screens.length; i++) { + x = Math.max(x, this._screens[i].x + this._screens[i].width); } var new_screen = { @@ -394,11 +281,8 @@ export default class Display { height: height, //client serverWidth: 0, //calculated serverHeight: 0, //calculated - x: 0, + x: x, y: 0, - relativePosition: relativePosition, - relativePositionX: relativePositionX, - relativePositionY: relativePositionY, pixelRatio: pixelRatio, containerHeight: containerHeight, containerWidth: containerWidth, @@ -418,15 +302,24 @@ export default class Display { } removeScreen(screenID) { + let removed = false; if (this._isPrimaryDisplay) { for (let i=1; i 0) { + this._screens[i].channel.postMessage({ eventType: "registered", screenIndex: i }); + } + } + return removed; } else { throw new Error("Secondary screens only allowed on primary display.") } @@ -857,6 +750,12 @@ export default class Display { this.flip(event.data.frameId, event.data.rectCnt); break; + case 'registered': + if (!this._isPrimaryDisplay) { + this._screens[0].screenIndex = event.data.screenIndex; + Log.Info(`Screen with index (${event.data.screenIndex}) successfully registered with the primary display.`); + } + break; } } } diff --git a/core/rfb.js b/core/rfb.js index 13a83001..42d6e4b8 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -207,6 +207,7 @@ export default class RFB extends EventTargetMixin { this._accumulatedWheelDeltaX = 0; this._accumulatedWheelDeltaY = 0; this.mouseButtonMapper = null; + this._mouseLastScreenIndex = 0; // Gesture state this._gestureLastTapTime = null; @@ -732,16 +733,68 @@ export default class RFB extends EventTargetMixin { // ===== PUBLIC METHODS ===== - attachSecondaryDisplay(relativePosition, relativePositionX, relativePositionY) { - this._display.relativePosition = relativePosition; - this._display.relativePositionX = relativePositionX; - this._display.relativePositionY = relativePositionY; - + attachSecondaryDisplay() { this._updateConnectionState('connecting'); this._registerSecondaryDisplay(); this._updateConnectionState('connected'); } + applyScreenPlan(screenPlan) { + if (this._isPrimaryDisplay) { + let fullPlan = this._display.getScreenSize(); + + //check plan for validity + let minX = Number.MAX_SAFE_INTEGER, minY = Number.MAX_SAFE_INTEGER; + let numScreensFound = 0; + + for (let i = 0; i < screenPlan.screens.length; i++) { + minX = Math.min(minX, screenPlan.screens[i].x); + minY = Math.min(minY, screenPlan.screens[i].y); + for (let z = 0; z < fullPlan.screens.length; z++) { + if (screenPlan.screens[i].screenID == fullPlan.screens[z].screenID) { + numScreensFound++; + } + } + } + if (minX !== 0 || minY !== 0) { + throw new Error("Screen plan invalid, improper coordinates provided."); + } + if (numScreensFound > fullPlan.screens.length) { + throw new Error("Screen plan contained more screens then there are registered.") + } else if (numScreensFound < fullPlan.screens.length) { + throw new Error("Screen plan contained fewer screens then there are registered.") + } + + this._display.applyScreenPlan(screenPlan); + const size = this._screenSize(); + RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); + this._updateContinuousUpdates(); + } + } + + getScreenPlan() { + let fullPlan = this._display.getScreenSize(); + let sanitizedPlan = { + screens: [], + serverWidth: fullPlan.serverWidth, + serverHeight: fullPlan.serverHeight + }; + + for (let i=0; i < fullPlan.screens.length; i++) { + sanitizedPlan.screens.push( + { + screenID: fullPlan.screens[i].screenID, + serverWidth: fullPlan.screens[i].serverWidth, + serverHeight: fullPlan.screens[i].serverHeight, + x: fullPlan.screens[i].x, + y: fullPlan.screens[i].y + } + ) + } + + return sanitizedPlan; + } + /* This function must be called after changing any properties that effect rendering quality */ @@ -1628,7 +1681,9 @@ export default class RFB extends EventTargetMixin { let message = { eventType: messageType, args: data, - screenId: this._display.screenId + screenId: this._display.screenId, + screenIndex: this._display.screenIndex, + mouseLastScreenIndex: this._mouseLastScreenIndex, } this._controlChannel.postMessage(message); } @@ -1638,10 +1693,11 @@ export default class RFB extends EventTargetMixin { // Secondary to Primary screen message switch (event.data.eventType) { case 'register': - this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.relativePositionX, event.data.relativePositionY, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); + this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth); const size = this._screenSize(); RFB.messages.setDesktopSize(this._sock, size, this._screenFlags); this._updateContinuousUpdates(); + this.dispatchEvent(new CustomEvent("screenregistered", {})); Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); break; case 'unregister': @@ -1654,9 +1710,11 @@ export default class RFB extends EventTargetMixin { } break; case 'pointerEvent': - let coords = this._display.getServerRelativeCoordinates(event.data.screenId, event.data.args[0], event.data.args[1]); + let coords = this._display.getServerRelativeCoordinates(event.data.screenIndex, event.data.args[0], event.data.args[1]); + this._mouseLastScreenIndex = event.data.screenIndex; event.data.args[0] = coords[0]; event.data.args[1] = coords[1]; + console.log(`screenIndex ${event.data.screenIndex}, x: ${coords[0]}, y: ${coords[1]}`); RFB.messages.pointerEvent(this._sock, ...event.data.args); break; case 'keyEvent': @@ -1665,6 +1723,9 @@ export default class RFB extends EventTargetMixin { case 'sendBinaryClipboard': RFB.messages.sendBinaryClipboard(this._sock, ...event.data.args); break; + // The following are primary to secondary messages that should be ignored on the primary + case 'updateCursor': + break; default: Log.Warn(`Unhandled message type (${event.data.eventType}) from control channel.`); } @@ -1672,7 +1733,9 @@ export default class RFB extends EventTargetMixin { // Primary to secondary screen message switch (event.data.eventType) { case 'updateCursor': - this._updateCursor(...event.data.args); + if (event.data.mouseLastScreenIndex === this._display.screenIndex) { + this._updateCursor(...event.data.args); + } break; } } @@ -1703,14 +1766,10 @@ export default class RFB extends EventTargetMixin { let message = { eventType: 'register', screenID: screen.screenID, - screenIndex: 1, width: screen.width, height: screen.height, x: 0, y: 0, - relativePosition: screen.relativePosition, - relativePositionX: screen.relativePositionX, - relativePositionY: screen.relativePositionY, pixelRatio: screen.pixelRatio, containerWidth: screen.containerWidth, containerHeight: screen.containerHeight, @@ -1818,6 +1877,7 @@ export default class RFB extends EventTargetMixin { this._canvas); } + this._mouseLastScreenIndex = this._display.screenIndex; this._setLastActive(); const mappedButton = this.mouseButtonMapper.get(ev.button); switch (ev.type) { @@ -1967,16 +2027,12 @@ export default class RFB extends EventTargetMixin { var rel_16_x = toSignedRelative16bit(x - this._pointerLockPos.x); var rel_16_y = toSignedRelative16bit(y - this._pointerLockPos.y); - //console.log("new_pos x" + x + ", y" + y); - //console.log("lock x " + this._pointerLockPos.x + ", y " + this._pointerLockPos.y); - //console.log("rel x " + rel_16_x + ", y " + rel_16_y); if (this._isPrimaryDisplay){ RFB.messages.pointerEvent(this._sock, rel_16_x, rel_16_y, mask); } else { this._proxyRFBMessage('pointerEvent', [ rel_16_x, rel_16_y, mask ]); } - // reset the cursor position to center this._mousePos = { x: this._pointerLockPos.x , y: this._pointerLockPos.y }; this._cursor.move(this._pointerLockPos.x, this._pointerLockPos.y); @@ -3436,9 +3492,9 @@ export default class RFB extends EventTargetMixin { const payload = this._sock.rQshiftStr(len); if (status) { - console.log("Unix relay subscription succeeded"); + Log.Info("Unix relay subscription succeeded"); } else { - console.log("Unix relay subscription failed, " + payload); + Log.Warn("Unix relay subscription failed, " + payload); } } @@ -3655,8 +3711,6 @@ export default class RFB extends EventTargetMixin { return false; } - console.log(`VMCursorUpdate x: ${hotx}, y: ${hoty}`); - this._updateCursor(rgba, hotx, hoty, w, h); return true; @@ -3881,6 +3935,7 @@ export default class RFB extends EventTargetMixin { rgbaPixels: rgba, hotx: hotx, hoty: hoty, w: w, h: h, }; + this._refreshCursor(); this._proxyRFBMessage('updateCursor', [ rgba, hotx, hoty, w, h ]); }