wired up mouse, keyboard, clipboard on second display

app/ui.js

@@ -1421,7 +1421,7 @@ const UI = {
         UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper();
         if (UI.rfb.videoQuality === 5) {
             UI.rfb.enableQOI = true;
-	}
+	    }
 
         //Only explicitly request permission to clipboard on browsers that support binary clipboard access
         if (supportsBinaryClipboard()) {

app/ui_screen.js

@@ -3,6 +3,7 @@ import * as WebUtil from "./webutil.js";
 import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS, supportsPointerLock }
     from '../core/util/browser.js';
 import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js";
+import * as Log from '../core/util/logging.js';
 
 const UI = {
     connected: false,
@@ -29,9 +30,11 @@ const UI = {
         document.getElementById('noVNC_connect_button').addEventListener('click', UI.connect);;
     },
 
-    getSetting(name, isBool) {
+    getSetting(name, isBool, default_value) {
-        const ctrl = document.getElementById('noVNC_setting_' + name);
         let val = WebUtil.readSetting(name);
+        if ((val === 'undefined' || val === null) && default_value !== 'undefined' && default_value !== null) {
+            val = default_value;
+        }
         if (typeof val !== 'undefined' && val !== null && isBool) {
             if (val.toString().toLowerCase() in {'0': 1, 'no': 1, 'false': 1}) {
                 val = false;
@@ -72,22 +75,27 @@ const UI = {
         UI.rfb.maxVideoResolutionY = parseInt(UI.getSetting('max_video_resolution_y'));
         UI.rfb.frameRate = parseInt(UI.getSetting('framerate'));
         UI.rfb.compressionLevel = parseInt(UI.getSetting('compression'));
-        UI.rfb.showDotCursor = UI.getSetting('show_dot');
+        UI.rfb.showDotCursor = UI.getSetting('show_dot', true);
         UI.rfb.idleDisconnect = UI.getSetting('idle_disconnect');
         UI.rfb.pointerRelative = UI.getSetting('pointer_relative');
         UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality'));
         UI.rfb.antiAliasing = UI.getSetting('anti_aliasing');
-        UI.rfb.clipboardUp = UI.getSetting('clipboard_up');
+        UI.rfb.clipboardUp = UI.getSetting('clipboard_up', true, true);
-        UI.rfb.clipboardDown = UI.getSetting('clipboard_down');
+        UI.rfb.clipboardDown = UI.getSetting('clipboard_down', true, true);
-        UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless');
+        UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless', true, true);
-        UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime');
+        UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime', true, false);
         UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless;
-        UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc');
+        UI.rfb.enableWebRTC = UI.getSetting('enable_webrtc', true, false);
-        UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi');
+        UI.rfb.enableHiDpi = UI.getSetting('enable_hidpi', true, false);
         UI.rfb.mouseButtonMapper = UI.initMouseButtonMapper();
         if (UI.rfb.videoQuality === 5) {
             UI.rfb.enableQOI = true;
         }
+
+        if (supportsBinaryClipboard()) {
+            // explicitly request permission to the clipboard
+            navigator.permissions.query({ name: "clipboard-read" }).then((result) => { Log.Debug('binary clipboard enabled') });
+        }
     },
 
     updateVisualState(state) {

core/display.js

@@ -121,6 +121,7 @@ export default class Display {
     // ===== PROPERTIES =====
 
     get screens() { return this._screens; }
+    get screenId() { return this._screenID; }
     
     get antiAliasing() { return this._antiAliasing; }
     set antiAliasing(value) {
@@ -161,6 +162,66 @@ export default class Display {
 
     // ===== PUBLIC METHODS =====
 
+    /*
+    Returns coordinates that are client relative when multiple monitors are in use
+    Returns an array with the following
+    0 - screen index
+    1 - screenId
+    2 - x
+    3 - y
+    */
+    getClientRelativeCoordinates(x, y) {
+        if (this._screens.length == 1) {
+            return [ 0, this._screenID, x, y ];
+        }
+        //TODO: The following logic will only support two monitors placed horizontally
+        let screenOrientation = this._screens[1].relativePosition;
+        let screenIdx = 0;
+        let screenId = this._screens[0].screenID;
+        if (screenOrientation == 0) {
+            if (x >= this._screens[1].x) {
+                x -= this._screens[1].x;
+                screenIdx  = 1;
+                screenId = this._screens[1].screenID;
+            }
+        } else if (screenOrientation == 2) {
+            if (x >= this._screens[0].x) {
+                x -= this._screens[0].x;
+            }
+        }
+        return [ screenIdx, screenId, x, y ];
+    }
+
+    /* 
+    Returns coordinates that are server relative when multiple monitors are in use
+    */
+    getServerRelativeCoordinates(screenId, x, y) {
+        // If this is the primary screen and only one screen, lets keep it simple
+        if (this._isPrimaryDisplay && this._screens.length == 1) {
+            return [x, y];
+        }
+
+        // Find the screen index by ID
+        let screenIdx = -1;
+        for (let i=0; i<this._screens.length; i++) {
+            if (screenId == this._screens[i].screenID) {
+                screenIdx = i;
+                break;
+            }
+        }
+
+        // If we didn't find the screen, log and return coords
+        if (screenIdx < 0) {
+            Log.Warn('Invalid screen ID presented for getServerRelativeCoordinates');
+            return [x, y]
+        }
+        
+        x += this._screens[screenIdx].x;
+        y += this._screens[screenIdx].y;
+
+        return [x, y];
+    }
+
     getScreenSize(resolutionQuality, max_width, max_height, hiDpi, disableLimit) {
         let data = {
             screens: null,
@@ -710,11 +771,10 @@ export default class Display {
                     rect.screenLocations = [ rect.screenLocations[event.data.screenLocationIndex] ]
                     rect.screenLocations[0].screenIndex = 0;
                     let pos = rect.screenLocations[0];
-                    
+                    //console.log(`${rect.type} Rect: x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`)
                     switch (rect.type) {
                         case 'copy':
                             //this.copyImage(rect.oldX, rect.oldY, pos.x, pos.y, rect.width, rect.height, rect.frame_id, true);
-                            console.log(`Copy Rect: src.x: ${rect.oldX}, src.y: ${rect.oldY}, x: ${pos.x}, y: ${pos.y}, w: ${rect.width}, h: ${rect.height}`)
                             this._asyncRenderQPush(rect);
                             break;
                         case 'fill':
@@ -726,6 +786,7 @@ export default class Display {
                             //this.blitImage(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true);
                             break;
                         case 'blitQ':
+                            
                             this._asyncRenderQPush(rect);
                             //this.blitQoi(pos.x, pos.y, rect.width, rect.height, rect.data, 0, rect.frame_id, true);
                             break;
@@ -783,7 +844,7 @@ export default class Display {
                 if (this._asyncFrameQueue[frameIx][1] !== 0) {
                     Log.Warn("Redundant flip rect, current rect_cnt: " + this._asyncFrameQueue[frameIx][1] + ", new rect_cnt: " + rect.rect_cnt );
                 }
-                this._asyncFrameQueue[frameIx][1] = rect.rect_cnt;
+                this._asyncFrameQueue[frameIx][1] += rect.rect_cnt;
                 if (rect.rect_cnt == 0) {
                     Log.Warn("Invalid rect count");
                 }  

core/rfb.js

@@ -820,7 +820,7 @@ export default class RFB extends EventTargetMixin {
             Log.Info("Sending key (" + (down ? "down" : "up") + "): keysym " + keysym + ", scancode " + scancode);
 
             if (this._isPrimaryDisplay) {
-                RFB.messages.QEMUExtendedKeyEvent(this._sock, [ keysym, down, scancode]);
+                RFB.messages.QEMUExtendedKeyEvent(this._sock, keysym, down, scancode);
             } else {
                 this._proxyRFBMessage('QEMUExtendedKeyEvent', [ keysym, down, scancode ])
             }
@@ -1617,19 +1617,22 @@ export default class RFB extends EventTargetMixin {
 
     _proxyRFBMessage(messageType, data) {
         let message = { 
-            messageType: messageType,
+            eventType: messageType,
-            data: data
+            args: data,
+            screenId: this._display.screenId
         }
         this._controlChannel.postMessage(message);
     }
 
     _handleControlMessage(event) {
         if (this._isPrimaryDisplay) {
+            // Secondary to Primary screen message
             switch (event.data.eventType) {
                 case 'register':
                     this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.relativePosition, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth);
                     const size = this._screenSize();
                     RFB.messages.setDesktopSize(this._sock, size, this._screenFlags);
+                    this._updateContinuousUpdates();
                     Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`);
                     break;
                 case 'unregister':
@@ -1640,6 +1643,28 @@ export default class RFB extends EventTargetMixin {
                     } else {
                         Log.Info(`Secondary monitor (${event.data.screenID}) not found.`);
                     }
+                    break;
+                case 'pointerEvent':
+                    let coords = this._display.getServerRelativeCoordinates(event.data.screenId, event.data.args[0], event.data.args[1]);
+                    event.data.args[0] = coords[0];
+                    event.data.args[1] = coords[1];
+                    RFB.messages.pointerEvent(this._sock, ...event.data.args);
+                    break;
+                case 'keyEvent':
+                    RFB.messages.keyEvent(this._sock, ...event.data.args);
+                    break;
+                case 'sendBinaryClipboard':
+                    RFB.messages.sendBinaryClipboard(this._sock, ...event.data.args);
+                    break;
+                default:
+                    Log.Warn(`Unhandled message type (${event.data.eventType}) from control channel.`);
+            }
+        } else {
+            // Primary to secondary screen message
+            switch (event.data.eventType) {
+                case 'updateCursor':
+                    this._updateCursor(...event.data.args);
+                    break;
             }
         }
         
@@ -1670,6 +1695,8 @@ export default class RFB extends EventTargetMixin {
                 channel: null
             }
             this._controlChannel.postMessage(message);
+
+            if (!this._viewOnly) { this._keyboard.grab(); }
         }
         
     }
@@ -3606,6 +3633,8 @@ export default class RFB extends EventTargetMixin {
             return false;
         }
 
+        console.log(`VMCursorUpdate x: ${hotx}, y: ${hoty}`);
+
         this._updateCursor(rgba, hotx, hoty, w, h);
 
         return true;
@@ -3831,6 +3860,7 @@ export default class RFB extends EventTargetMixin {
             hotx: hotx, hoty: hoty, w: w, h: h,
         };
         this._refreshCursor();
+        this._proxyRFBMessage('updateCursor', [ rgba, hotx, hoty, w, h ]);
     }
 
     _shouldShowDotCursor() {