From 1753425874f8fed324db514e99ee4daccda918ba Mon Sep 17 00:00:00 2001 From: Matt McClaskey Date: Tue, 3 May 2022 10:37:22 -0400 Subject: [PATCH] Pointer lock api (#29) Add pointer lock and relative cursor position support. Game mode enables both pointer lock and relative cursor positions. --- app/error-handler.js | 6 ++ app/images/gamepad.png | Bin 0 -> 2098 bytes app/images/pointer.svg | 78 ++++++++++++++++ app/ui.js | 206 +++++++++++++++++++++++++++++++++++------ core/encodings.js | 1 + core/rfb.js | 170 +++++++++++++++++++++++++++++++--- core/util/browser.js | 48 ++++++++++ core/util/cursor.js | 57 ++++++++---- core/util/int.js | 32 ++++++- docs/API.md | 38 ++++++++ tests/test.rfb.js | 77 +++++++++++++++ vnc.html | 27 +++++- 12 files changed, 682 insertions(+), 58 deletions(-) create mode 100644 app/images/gamepad.png create mode 100644 app/images/pointer.svg diff --git a/app/error-handler.js b/app/error-handler.js index c65dcd1a..cf966f47 100644 --- a/app/error-handler.js +++ b/app/error-handler.js @@ -26,6 +26,12 @@ return false; } + // Skip allowed errors + let allowedErrors = [ "The user has exited the lock before this request was completed." ]; + if (event.message && allowedErrors.includes(event.message)) { + return false; + } + let div = document.createElement("div"); div.classList.add('noVNC_message'); div.appendChild(document.createTextNode(event.message)); diff --git a/app/images/gamepad.png b/app/images/gamepad.png new file mode 100644 index 0000000000000000000000000000000000000000..5ea9f1e98fa0404252317c721f6d91bfb2368dff GIT binary patch literal 2098 zcmeAS@N?(olHy`uVBq!ia0vp^k|4~%1|*NXY)uAIjKx9jP7LeL$-D$|4y8tTruq6Z zXaU(A4D5_T49p-UK*+!-#lQ+?GcY7EO2gS%j2ciiOh7e;3_y}W5QvKyjlgUXAiJeK zlYs@QcLtCK0S_PsnhK!o>zjI0cdtc*+*3{Aicpdj-C zMudA7Fu~1~T)>QAgAAW)RJj01aTa()7Bet#3xhBt!>lsd^Q;1t47vHWgMtW^QUpqC!P(PF}H9g{=};g%ywu64qBz04piUwpEJo4N!2- zFG^J~(=*UBP_pAvP*AWbN=dT{a&d!d2l8x{GD=Dctn~HE%ggo3jrH=2()A53EiLs8 zjP#9+bb%^#i!1X=5-W7`ij^UTz|3(;Elw`VEGWs$&r<-Io0ybeT4JlD1hNPYAnr`9 z$VAwbR}A$Q(1ZFQ8GS=N1AT1TfS!fB4@Fl+VgXJ~C7EeB>;lQ))RvMAx2mADCKQE)+Ga;lx74cIftLWq!ns0Jsa2$xKt ziN(NBvokV<>Oq&lsxcCmMkEO&jljUOaxO{*CEAeu{2V*3 zNva(vwd)!f=^B}az*0LzJCYhC-JpQB3Q8@`&nX3^j?lc!5<4RseQd($x}5WK3yM;U z!NFqy)q*UBuHL^W6Bs>-AZtJ|0JZ{I3|)N$!dZ6aFgGC!qHBo6rU6+HNkdAqRdH!j zR%&tyIJdzw0XW+^C8p~m3ESvnsVr zrl*TzhzIZ65bKP|fg*nYW(4&Q(-|+i9q25g&PZB zD7bBqkP;}JV%54zLsF)vQ7tzo$?%Ee>;yTJt(EOIzJKywahjj>x3;>$J^k~qbC%z0 z{{LO?vT64}Z?+{#!M|(v37<{-?6x=VdT&pUjpkIZ&yOEJ{#m!?Zr*)+GqWa+*lH#HJ_K4S5~QD@x_kB{4KY0i%M0v+099U#qpIW-VhsA+0yvJ9cG=m&P?c5iaeE871$`#Wrt`KAIHBenaPQ!Y?-kfeW8X zt4roQ;$wdHxpMl=9J}rZh5g4rGh8W^+AOjE!S{rE7iRr$Tv8E&>z7@A`EuT&Jrhr+ zkro6eb!Kyz81|5sZw*YA1h z)~L0I-D{6$8SFpd^j56us$TLUei8QCbMp?(QG9SDj!9`{gRE*b`%kHl@-2>YHylm6 zdHU+Tpj@-tjx}HAv!n>wTz7mpHOH)4YPCiI zT{!ak>#BeiCz>6dvw%RJ*_i9p(@ombPuDj6VUh}8!6CKoQ{|gBU0vNwjeAoMCVWsi zz|qry^v|eqc^=ZtmMj5%&%qI>g1^qzCTG<&5(Do&MkRimV++)@`PKRK2DvZ^6~zjnKNd1Ox9K7WpfrZf0J6z zbN{>b&t1L+Qr~X5X*ApxIU@aK#+$b)^W@4nV_FVAytJe7`iXayt+IUWLi(?DcgDoI zUn*XFafZ?Zze`zL<*sFzY%jdb_n={O)n2 + + + + + + + + + image/svg+xml + + + + + + + + + \ No newline at end of file diff --git a/app/ui.js b/app/ui.js index 41e31c64..e60b4809 100644 --- a/app/ui.js +++ b/app/ui.js @@ -34,7 +34,7 @@ import "core-js/stable"; import "regenerator-runtime/runtime"; import * as Log from '../core/util/logging.js'; import _, { l10n } from './localization.js'; -import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS } +import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS, supportsPointerLock } from '../core/util/browser.js'; import { setCapture, getPointerEvent } from '../core/util/events.js'; import KeyTable from "../core/input/keysym.js"; @@ -47,6 +47,7 @@ const PAGE_TITLE = "KasmVNC"; var delta = 500; var lastKeypressTime = 0; +var lastKeypressCode = -1; var currentEventCount = -1; var idleCounter = 0; @@ -350,6 +351,10 @@ const UI = { document.getElementById("noVNC_view_drag_button") .addEventListener('click', UI.toggleViewDrag); + document + .getElementById("noVNC_setting_pointer_lock") + .addEventListener("click", UI.togglePointerLock); + document.getElementById("noVNC_control_bar_handle") .addEventListener('mousedown', UI.controlbarHandleMouseDown); document.getElementById("noVNC_control_bar_handle") @@ -415,6 +420,8 @@ const UI = { .addEventListener('click', UI.sendEsc); document.getElementById("noVNC_send_ctrl_alt_del_button") .addEventListener('click', UI.sendCtrlAltDel); + document.getElementById("noVNC_game_mode_button") + .addEventListener("click", UI.toggleRelativePointer) }, addMachineHandlers() { @@ -527,6 +534,7 @@ const UI = { UI.addSettingChangeHandler('clipboard_seamless'); UI.addSettingChangeHandler('clipboard_up'); UI.addSettingChangeHandler('clipboard_down'); + UI.addSettingChangeHandler('toggle_control_panel'); UI.addSettingChangeHandler('virtual_keyboard_visible'); UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls); UI.addSettingChangeHandler('enable_ime'); @@ -611,6 +619,7 @@ const UI = { UI.updatePowerButton(); UI.keepControlbar(); } + //UI.updatePointerLockButton(); // State change closes dialogs as they may not be relevant // anymore @@ -637,7 +646,12 @@ const UI = { }, - showStatus(text, statusType, time) { + showStatus(text, statusType, time, kasm = false) { + // If inside the full Kasm CDI framework, don't show messages unless explicitly told to + if (WebUtil.isInsideKasmVDI() && !kasm) { + return; + } + const statusElem = document.getElementById('noVNC_status'); if (typeof statusType === 'undefined') { @@ -1307,6 +1321,8 @@ const UI = { UI.rfb.addEventListener("bottleneck_stats", UI.bottleneckStatsRecieve); UI.rfb.addEventListener("bell", UI.bell); UI.rfb.addEventListener("desktopname", UI.updateDesktopName); + UI.rfb.addEventListener("inputlock", UI.inputLockChanged); + UI.rfb.addEventListener("inputlockerror", UI.inputLockError); UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts'); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; @@ -1327,6 +1343,7 @@ const UI = { UI.rfb.compressionLevel = parseInt(UI.getSetting('compression')); UI.rfb.showDotCursor = UI.getSetting('show_dot'); UI.rfb.idleDisconnect = UI.getSetting('idle_disconnect'); + UI.rfb.pointerRelative = UI.getSetting('pointer_relative'); UI.rfb.videoQuality = parseInt(UI.getSetting('video_quality')); UI.rfb.antiAliasing = UI.getSetting('anti_aliasing'); UI.rfb.clipboardUp = UI.getSetting('clipboard_up'); @@ -1394,23 +1411,23 @@ const UI = { document.getElementById('noVNC_status').style.visibility = "visible"; } - // Send an event to the parent document (kasm app) to toggle the control panel when ctl is double clicked - if (UI.getSetting('toggle_control_panel', false)) { - - document.addEventListener('keyup', function (event) { - // CTRL and the various implementations of the mac command key - if ([17, 224, 91, 93].indexOf(event.keyCode) > -1) { - var thisKeypressTime = new Date(); - - if (thisKeypressTime - lastKeypressTime <= delta) { - UI.toggleNav(); - thisKeypressTime = 0; + //key events for KasmVNC control + document.addEventListener('keyup', function (event) { + if (event.ctrlKey && event.shiftKey) { + switch(event.keyCode) { + case 49: + UI.toggleNav(); + break; + case 50: + UI.toggleRelativePointer(); + break; + case 51: + UI.togglePointerLock(); + break; } + } - lastKeypressTime = thisKeypressTime; - } - }, true); - } + }, true); }, disconnect() { @@ -1523,18 +1540,19 @@ const UI = { UI.showStatus(msg, 'error'); }, - /* - Menu.js Additions - */ - receiveMessage(event) { - //TODO: UNCOMMENT FOR PRODUCTION - //if (event.origin !== "https://kasmweb.com") - // return; + //send message to parent window + sendMessage(name, value) { + if (WebUtil.isInsideKasmVDI()) { + parent.postMessage({ action: name, value: value }, '*' ); + } + }, + //receive message from parent window + receiveMessage(event) { if (event.data && event.data.action) { switch (event.data.action) { case 'clipboardsnd': - if (UI.rfb.clipboardUp) { + if (UI.rfb && UI.rfb.clipboardUp) { UI.rfb.clipboardPasteFrom(event.data.value); } break; @@ -1542,6 +1560,26 @@ const UI = { UI.forceSetting('video_quality', parseInt(event.data.value), false); UI.updateQuality(); break; + case 'enable_game_mode': + if (UI.rfb && !UI.rfb.pointerRelative) { + UI.toggleRelativePointer(); + } + break; + case 'disable_game_mode': + if (UI.rfb && UI.rfb.pointerRelative) { + UI.toggleRelativePointer(); + } + break; + case 'enable_pointer_lock': + if (UI.rfb && !UI.rfb.pointerLock) { + UI.togglePointerLock(); + } + break; + case 'disable_pointer_lock': + if (UI.rfb && UI.rfb.pointerLock) { + UI.togglePointerLock(); + } + break; case 'show_keyboard_controls': if (!UI.getSetting('virtual_keyboard_visible')) { UI.forceSetting('virtual_keyboard_visible', true, false); @@ -1575,7 +1613,15 @@ const UI = { }, toggleNav(){ - parent.postMessage({ action: 'togglenav', value: null}, '*' ); + if (WebUtil.isInsideKasmVDI()) { + parent.postMessage({ action: 'togglenav', value: null}, '*' ); + } else { + UI.toggleControlbar(); + UI.keepControlbar(); + UI.activateControlbar(); + UI.controlbarGrabbed = false; + UI.showControlbarHint(false); + } }, clipboardRx(event) { @@ -1678,6 +1724,7 @@ const UI = { document.getElementById('noVNC_fullscreen_button') .classList.remove("noVNC_selected"); } + UI.updatePointerLockButton(); }, /* ------^------- @@ -1730,6 +1777,76 @@ const UI = { UI.updateViewDrag(); }, + /* ------^------- + * /VIEW CLIPPING + * ============== + * POINTER LOCK + * ------v------*/ + + updatePointerLockButton() { + // Only show the button if the pointer lock API is properly supported + // AND in fullscreen. + if ( + UI.connected && + (document.pointerLockElement !== undefined || + document.mozPointerLockElement !== undefined) + ) { + document + .getElementById("noVNC_setting_pointer_lock") + .classList.remove("noVNC_hidden"); + document + .getElementById("noVNC_game_mode_button") + .classList.remove("noVNC_hidden"); + } else { + document + .getElementById("noVNC_setting_pointer_lock") + .classList.add("noVNC_hidden"); + document + .getElementById("noVNC_game_mode_button") + .classList.add("noVNC_hidden"); + } + }, + + togglePointerLock() { + if (!supportsPointerLock()) { + UI.showStatus('Your browser does not support pointer lock.', 'info', 1500, true); + //force pointer lock in UI to false and disable control + UI.forceSetting('pointer_lock', false, true); + } else { + UI.rfb.pointerLock = !UI.rfb.pointerLock; + if (UI.getSetting('pointer_lock') !== UI.rfb.pointerLock) { + UI.forceSetting('pointer_lock', UI.rfb.pointerLock, false); + } + } + }, + + toggleRelativePointer(event=null, forcedToggleValue=null) { + if (!supportsPointerLock()) { + UI.showStatus('Your browser does not support pointer lock.', 'info', 1500, true); + return; + } + + var togglePosition = !UI.rfb.pointerRelative; + + if (UI.rfb.pointerLock !== togglePosition) { + UI.rfb.pointerLock = togglePosition; + } + if (UI.rfb.pointerRelative !== togglePosition) { + UI.rfb.pointerRelative = togglePosition; + } + + if (togglePosition) { + document.getElementById('noVNC_game_mode_button').classList.add("noVNC_selected"); + } else { + document.getElementById('noVNC_game_mode_button').classList.remove("noVNC_selected"); + UI.forceSetting('pointer_lock', false, false); + } + + UI.sendMessage('enable_game_mode', togglePosition); + UI.sendMessage('enable_pointer_lock', togglePosition); + + }, + /* ------^------- * /VIEW CLIPPING * ============== @@ -2161,6 +2278,8 @@ const UI = { .classList.add('noVNC_hidden'); document.getElementById('noVNC_clipboard_button') .classList.add('noVNC_hidden'); + document.getElementById('noVNC_game_mode_button') + .classList.add('noVNC_hidden'); } else { document.getElementById('noVNC_keyboard_button') .classList.remove('noVNC_hidden'); @@ -2168,6 +2287,8 @@ const UI = { .classList.remove('noVNC_hidden'); document.getElementById('noVNC_clipboard_button') .classList.remove('noVNC_hidden'); + document.getElementById('noVNC_game_mode_button') + .classList.remove('noVNC_hidden'); } }, @@ -2186,6 +2307,39 @@ const UI = { document.title = e.detail.name + " - " + PAGE_TITLE; }, + inputLockChanged(e) { + var pointer_lock_el = document.getElementById("noVNC_setting_pointer_lock"); + var pointer_rel_el = document.getElementById("noVNC_game_mode_button"); + + if (e.detail.pointer) { + pointer_lock_el.checked = true; + UI.sendMessage('enable_pointer_lock', true); + UI.closeControlbar(); + UI.showStatus('Press Esc Key to Exit Pointer Lock Mode', 'warn', 5000, true); + } else { + //If in game mode + if (UI.rfb.pointerRelative) { + UI.showStatus('Game Mode paused, click on screen to resume Game Mode.', 'warn', 5000, true); + } else { + UI.forceSetting('pointer_lock', false, false); + document.getElementById('noVNC_game_mode_button') + .classList.remove("noVNC_selected"); + UI.sendMessage('enable_pointer_lock', false); + } + } + }, + + inputLockError(e) { + UI.showStatus('Unable to enter pointer lock mode.', 'warn', 5000, true); + UI.rfb.pointerRelative = false; + + document.getElementById('noVNC_game_mode_button').classList.remove("noVNC_selected"); + UI.forceSetting('pointer_lock', false, false); + + UI.sendMessage('enable_game_mode', false); + UI.sendMessage('enable_pointer_lock', false); + }, + bell(e) { if (WebUtil.getConfigVar('bell', 'on') === 'on') { const promise = document.getElementById('noVNC_bell').play(); diff --git a/core/encodings.js b/core/encodings.js index b1bb12b2..d2f081ca 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -54,6 +54,7 @@ export const encodings = { pseudoEncodingVideoOutTimeLevel100: -1887, pseudoEncodingVMwareCursor: 0x574d5664, + pseudoEncodingVMwareCursorPosition: 0x574d5666, pseudoEncodingExtendedClipboard: 0xc0a1e5ce }; diff --git a/core/rfb.js b/core/rfb.js index 41e8c4ce..74611321 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -33,6 +33,7 @@ import RREDecoder from "./decoders/rre.js"; import HextileDecoder from "./decoders/hextile.js"; import TightDecoder from "./decoders/tight.js"; import TightPNGDecoder from "./decoders/tightpng.js"; +import { toSignedRelative16bit } from './util/int.js'; // How many seconds to wait for a disconnect to finish const DISCONNECT_TIMEOUT = 3; @@ -42,7 +43,7 @@ var _videoQuality = 2; var _enableWebP = false; // Minimum wait (ms) between two mouse moves -const MOUSE_MOVE_DELAY = 17; +const MOUSE_MOVE_DELAY = 17; // Wheel thresholds let WHEEL_LINE_HEIGHT = 19; // Pixels for one line step (on Windows) @@ -170,6 +171,9 @@ export default class RFB extends EventTargetMixin { this._mousePos = {}; this._mouseButtonMask = 0; this._mouseLastMoveTime = 0; + this._pointerLock = false; + this._pointerLockPos = { x: 0, y: 0 }; + this._pointerRelativeEnabled = false; this._mouseLastPinchAndZoomTime = 0; this._viewportDragging = false; this._viewportDragPos = {}; @@ -189,6 +193,8 @@ export default class RFB extends EventTargetMixin { focusCanvas: this._focusCanvas.bind(this), windowResize: this._windowResize.bind(this), handleMouse: this._handleMouse.bind(this), + handlePointerLockChange: this._handlePointerLockChange.bind(this), + handlePointerLockError: this._handlePointerLockError.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), }; @@ -333,6 +339,43 @@ export default class RFB extends EventTargetMixin { // ===== PROPERTIES ===== + get pointerLock() { return this._pointerLock; } + set pointerLock(value) { + if (!this._pointerLock) { + if (this._canvas.requestPointerLock) { + this._canvas.requestPointerLock(); + this._pointerLockChanging = true; + } else if (this._canvas.mozRequestPointerLock) { + this._canvas.mozRequestPointerLock(); + this._pointerLockChanging = true; + } + } else { + if (window.document.exitPointerLock) { + window.document.exitPointerLock(); + this._pointerLockChanging = true; + } else if (window.document.mozExitPointerLock) { + window.document.mozExitPointerLock(); + this._pointerLockChanging = true; + } + } + } + + get pointerRelative() { return this._pointerRelativeEnabled; } + set pointerRelative(value) + { + this._pointerRelativeEnabled = value; + if (value) { + let max_w = ((this._display.scale === 1) ? this._fbWidth : (this._fbWidth * this._display.scale)); + let max_h = ((this._display.scale === 1) ? this._fbHeight : (this._fbHeight * this._display.scale)); + this._pointerLockPos.x = Math.floor(max_w / 2); + this._pointerLockPos.y = Math.floor(max_h / 2); + + // reset the cursor position to center + this._mousePos = { x: this._pointerLockPos.x , y: this._pointerLockPos.y }; + this._cursor.move(this._pointerLockPos.x, this._pointerLockPos.y); + } + } + get keyboard() { return this._keyboard; } get clipboardBinary() { return this._clipboardMode; } @@ -748,12 +791,10 @@ export default class RFB extends EventTargetMixin { focus() { this._keyboard.focus(); - //this._canvas.focus(); } blur() { this._keyboard.blur(); - //this._canvas.blur(); } clipboardPasteFrom(text) { @@ -915,6 +956,15 @@ export default class RFB extends EventTargetMixin { // reason so we have to explicitly block it this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse); + // Pointer Lock listeners need to be installed in document instead of the canvas. + if (document.onpointerlockchange !== undefined) { + document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange, false); + document.addEventListener('pointerlockerror', this._eventHandlers.handlePointerLockError, false); + } else if (document.onmozpointerlockchange !== undefined) { + document.addEventListener('mozpointerlockchange', this._eventHandlers.handlePointerLockChange, false); + document.addEventListener('mozpointerlockerror', this._eventHandlers.handlePointerLockError, false); + } + // Wheel events this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); @@ -938,6 +988,13 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse); this._canvas.removeEventListener('click', this._eventHandlers.handleMouse); this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse); + if (document.onpointerlockchange !== undefined) { + document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); + document.removeEventListener('pointerlockerror', this._eventHandlers.handlePointerLockError); + } else if (document.onmozpointerlockchange !== undefined) { + document.removeEventListener('mozpointerlockchange', this._eventHandlers.handlePointerLockChange); + document.removeEventListener('mozpointerlockerror', this._eventHandlers.handlePointerLockError); + } this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); @@ -976,11 +1033,18 @@ export default class RFB extends EventTargetMixin { value: null }, "*"); + // Re-enable pointerLock if relative cursor is enabled + // pointerLock must come from user initiated event + if (!this._pointerLock && this._pointerRelativeEnabled) { + this.pointerLock = true; + } + if (!this.focusOnClick) { return; } this.focus(); + } _setDesktopName(name) { @@ -1311,8 +1375,34 @@ export default class RFB extends EventTargetMixin { return; } - let pos = clientToElement(ev.clientX, ev.clientY, + let pos; + if (this._pointerLock && !this._pointerRelativeEnabled) { + let max_w = ((this._display.scale === 1) ? this._fbWidth : (this._fbWidth * this._display.scale)); + let max_h = ((this._display.scale === 1) ? this._fbHeight : (this._fbHeight * this._display.scale)); + pos = { + x: this._mousePos.x + ev.movementX, + y: this._mousePos.y + ev.movementY, + }; + if (pos.x < 0) { + pos.x = 0; + } else if (pos.x > max_w) { + pos.x = max_w; + } + if (pos.y < 0) { + pos.y = 0; + } else if (pos.y > max_h) { + pos.y = max_h; + } + this._cursor.move(pos.x, pos.y); + } else if (this._pointerLock && this._pointerRelativeEnabled) { + pos = { + x: this._mousePos.x + ev.movementX, + y: this._mousePos.y + ev.movementY, + }; + } else { + pos = clientToElement(ev.clientX, ev.clientY, this._canvas); + } switch (ev.type) { case 'mousedown': @@ -1428,12 +1518,54 @@ export default class RFB extends EventTargetMixin { this._mouseLastMoveTime = Date.now(); } + _handlePointerLockChange(env) { + if ( + document.pointerLockElement === this._canvas || + document.mozPointerLockElement === this._canvas + ) { + this._pointerLock = true; + this._cursor.setEmulateCursor(true); + } else { + this._pointerLock = false; + this._cursor.setEmulateCursor(false); + } + this.dispatchEvent(new CustomEvent( + "inputlock", + { detail: { pointer: this._pointerLock }, })); + } + + _handlePointerLockError() { + this._pointerLockChanging = false; + this.dispatchEvent(new CustomEvent( + "inputlockerror", + { detail: { pointer: this._pointerLock }, })); + } + _sendMouse(x, y, mask) { if (this._rfbConnectionState !== 'connected') { return; } if (this._viewOnly) { return; } // View only, skip mouse events - RFB.messages.pointerEvent(this._sock, this._display.absX(x), + if (this._pointerLock && this._pointerRelativeEnabled) { + + // Use releative cursor position + var rel_16_x = toSignedRelative16bit(x - this._pointerLockPos.x); + var rel_16_y = toSignedRelative16bit(y - this._pointerLockPos.y); + + //console.log("new_pos x" + x + ", y" + y); + //console.log("lock x " + this._pointerLockPos.x + ", y " + this._pointerLockPos.y); + //console.log("rel x " + rel_16_x + ", y " + rel_16_y); + + RFB.messages.pointerEvent(this._sock, rel_16_x, + rel_16_y, mask); + + // reset the cursor position to center + this._mousePos = { x: this._pointerLockPos.x , y: this._pointerLockPos.y }; + this._cursor.move(this._pointerLockPos.x, this._pointerLockPos.y); + } else { + RFB.messages.pointerEvent(this._sock, this._display.absX(x), this._display.absY(y), mask); + } + } _sendScroll(x, y, dX, dY) { @@ -2254,16 +2386,16 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingVideoScalingLevel0 + this.videoScaling); encs.push(encodings.pseudoEncodingFrameRateLevel10 + this.frameRate - 10); encs.push(encodings.pseudoEncodingMaxVideoResolution); - // preferBandwidth choses preset settings. Since we expose all the settings, lets not pass this + + // preferBandwidth choses preset settings. Since we expose all the settings, lets not pass this if (this.preferBandwidth) // must be last - server processes in reverse order encs.push(encodings.pseudoEncodingPreferBandwidth); - if (supportsCursorURIs && this._fbDepth == 24) { - if (this.preferLocalCursor || !isTouchDevice) { - encs.push(encodings.pseudoEncodingVMwareCursor); - encs.push(encodings.pseudoEncodingCursor); - } + if (this._fbDepth == 24) { + encs.push(encodings.pseudoEncodingVMwareCursor); + encs.push(encodings.pseudoEncodingCursor); } + encs.push(encodings.pseudoEncodingVMwareCursorPosition); RFB.messages.clientEncodings(this._sock, encs); } @@ -2795,6 +2927,9 @@ export default class RFB extends EventTargetMixin { case encodings.pseudoEncodingVMwareCursor: return this._handleVMwareCursor(); + case encodings.pseudoEncodingVMwareCursorPosition: + return this._handleVMwareCursorPosition(); + case encodings.pseudoEncodingCursor: return this._handleCursor(); @@ -2933,6 +3068,19 @@ export default class RFB extends EventTargetMixin { return true; } + _handleVMwareCursorPosition() { + const x = this._FBU.x; + const y = this._FBU.y; + + if (this._pointerLock) { + // Only attempt to match the server's pointer position if we are in + // pointer lock mode. + this._mousePos = { x: x, y: y }; + } + + return true; + } + _handleCursor() { const hotx = this._FBU.x; // hotspot-x const hoty = this._FBU.y; // hotspot-y diff --git a/core/util/browser.js b/core/util/browser.js index 23798f39..23bf6cb5 100644 --- a/core/util/browser.js +++ b/core/util/browser.js @@ -97,6 +97,47 @@ export function isSafari() { navigator.userAgent.indexOf('Chrome') === -1); } +// Returns IE version number if IE or older Edge browser +export function isIE() { + var ua = window.navigator.userAgent; + + // Test values; Uncomment to check result & + + // IE 10 + // ua = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'; + + // IE 11 + // ua = 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'; + + // Edge 12 (Spartan) + // ua = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0'; + + // Edge 13 + // ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586'; + + var msie = ua.indexOf('MSIE '); + var ie_ver = false; + if (msie > 0) { + // IE 10 or older => return version number + ie_ver = parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10); + } + + var trident = ua.indexOf('Trident/'); + if (trident > 0) { + // IE 11 => return version number + var rv = ua.indexOf('rv:'); + ie_ver = parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10); + } + + var edge = ua.indexOf('Edge/'); + if (edge > 0) { + // Edge (IE 12+) => return version number + ie_ver = parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10); + } + + return ie_ver; +} + export function isChromiumBased() { return (!!window.chrome); } @@ -111,3 +152,10 @@ export function supportsBinaryClipboard() { return (navigator.clipboard && typeof navigator.clipboard.read === "function"); } +export function supportsPointerLock() { + //Older versions of edge do support browser lock, but seems to not behave as expected + //Disable on browsers that don't fully support or work as expected + if (isIOS() || isIE()) { return false; } + return (document.exitPointerLock); +} + diff --git a/core/util/cursor.js b/core/util/cursor.js index 12bcceda..6d5200c1 100644 --- a/core/util/cursor.js +++ b/core/util/cursor.js @@ -6,21 +6,19 @@ import { supportsCursorURIs, isTouchDevice } from './browser.js'; -const useFallback = !supportsCursorURIs || isTouchDevice; +const needsFallback = !supportsCursorURIs || isTouchDevice; export default class Cursor { constructor() { this._target = null; this._canvas = document.createElement('canvas'); - - if (useFallback) { - this._canvas.style.position = 'fixed'; - this._canvas.style.zIndex = '65535'; - this._canvas.style.pointerEvents = 'none'; - // Can't use "display" because of Firefox bug #1445997 - this._canvas.style.visibility = 'hidden'; - } + this._canvas.style.position = 'fixed'; + this._canvas.style.zIndex = '65535'; + this._canvas.style.pointerEvents = 'none'; + // Can't use "display" because of Firefox bug #1445997 + this._canvas.style.visibility = 'hidden'; + this._useFallback = needsFallback; this._position = { x: 0, y: 0 }; this._hotSpot = { x: 0, y: 0 }; @@ -40,9 +38,15 @@ export default class Cursor { this._target = target; - if (useFallback) { - document.body.appendChild(this._canvas); - + + document.body.appendChild(this._canvas); + + if (needsFallback) { + // Only add the event listeners if this will be responsible for + // rendering the cursor all the time. Otherwise, the cursor will + // only be rendered then the forced emulation is turned on, and + // that doesn't require this class to be adjusting the cursor + // position. const options = { capture: true, passive: true }; this._target.addEventListener('mouseover', this._eventHandlers.mouseover, options); this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options); @@ -58,16 +62,16 @@ export default class Cursor { return; } - if (useFallback) { + if (needsFallback) { const options = { capture: true, passive: true }; this._target.removeEventListener('mouseover', this._eventHandlers.mouseover, options); this._target.removeEventListener('mouseleave', this._eventHandlers.mouseleave, options); this._target.removeEventListener('mousemove', this._eventHandlers.mousemove, options); this._target.removeEventListener('mouseup', this._eventHandlers.mouseup, options); - - document.body.removeChild(this._canvas); } + document.body.removeChild(this._canvas); + this._target = null; } @@ -91,9 +95,10 @@ export default class Cursor { ctx.clearRect(0, 0, w, h); ctx.putImageData(img, 0, 0); - if (useFallback) { + if (this._useFallback || needsFallback) { this._updatePosition(); - } else { + } + if (!needsFallback) { let url = this._canvas.toDataURL(); this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; } @@ -112,7 +117,7 @@ export default class Cursor { // Mouse events might be emulated, this allows // moving the cursor in such cases move(clientX, clientY) { - if (!useFallback) { + if (!this._useFallback) { return; } // clientX/clientY are relative the _visual viewport_, @@ -130,6 +135,22 @@ export default class Cursor { this._updateVisibility(target); } + // Force the use of cursor emulation. This is needed when the pointer lock + // is in use, since the browser will not render the cursor. + setEmulateCursor(emulate) { + if (needsFallback) { + // We need to use the fallback all the time, so we shouldn't update + // the fallback flag. + return; + } + this._useFallback = emulate; + if (this._useFallback) { + this._showCursor(); + } else { + this._hideCursor(); + } + } + _handleMouseOver(event) { // This event could be because we're entering the target, or // moving around amongst its sub elements. Let the move handler diff --git a/core/util/int.js b/core/util/int.js index 79c9f724..40521ce1 100644 --- a/core/util/int.js +++ b/core/util/int.js @@ -15,12 +15,40 @@ export function toSigned32bit(toConvert) { } /* - * Fast hashing function with low entropy, not for security uses. +* Converts a signed 32bit integer to a signed 16bit int +* Uses second most significant bit to represent it is relative */ +export function toSignedRelative16bit(toConvert) { + // TODO: move these so they are not computed with every func call + var negmask16 = 1 << 15; + var negmask32 = 1 << 31; + var relmask16 = 1 << 14; + + var converted16 = toConvert | 0; + + // number is negative + if ((toConvert & negmask32) != 0) { + // clear the 32bit negative bit + // not neccessary because the last 16bits will get dropped anyway + converted16 *= -1; + + // set the 16bit negative bit + converted16 |= negmask16; + // set the relative bit + converted16 |= relmask16; + } else { + // set the relative bit + converted16 |= relmask16; + } + + return converted16; +} + +/* Fast hashing function with low entropy */ export function hashUInt8Array(data) { let h; for (let i = 0; i < data.length; i++) { h = Math.imul(31, h) + data[i] | 0; } return h; -} \ No newline at end of file +} diff --git a/docs/API.md b/docs/API.md index aa5aea7a..af601aba 100644 --- a/docs/API.md +++ b/docs/API.md @@ -113,6 +113,10 @@ protocol stream. - The `capabilities` event is fired when `RFB.capabilities` is updated. +[`inputlock`](#inputlock) + - The `inputlock` event is fired when an input lock is acquired (or released) + by the canvas. + ### Methods [`RFB.disconnect()`](#rfbdisconnect) @@ -146,6 +150,10 @@ protocol stream. [`RFB.clipboardPasteFrom()`](#rfbclipboardPasteFrom) - Send clipboard contents to server. +[`inputlock`](#inputlock) + - The `inputlock` event is fired when an input lock is acquired (or released) + by the canvas. + ### Details #### RFB() @@ -262,6 +270,15 @@ The `capabilities` event is fired whenever an entry is added or removed from `RFB.capabilities`. The `detail` property is an `Object` with the property `capabilities` containing the new value of `RFB.capabilities`. +#### inputlock + +The `inputlock` event is fired after a request to acquire an input lock or +whenever the state of the canvas' input lock has changed, the latter typically +occurs because the lock was released by the user pressing the ESC key or +performing a browser-specific gesture. The `detail` property is an `Object` +with the property `pointer` containing whether the Pointer Lock is currently +held or not. + #### RFB.disconnect() The `RFB.disconnect()` method is used to disconnect from the currently @@ -383,3 +400,24 @@ to the remote server. **`text`** - A `DOMString` specifying the clipboard data to send. + +#### RFB.requestInputLock() + +The `RFB.requestInputLock()` method is used to request that the RFB canvas hold +an input lock. An `inputlock` event will be fired with the result of the +acquisition of the requested locks. + +##### Syntax + + RFB.requestInputLock( { pointer: true } ); + +###### Parameters + +**`pointer`** + - Requests to acquire a [Pointer + Lock](https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API), + which hides the local mouse cursor and provides relative motion events. + This must be called directly from an event handler where a user has + directly interacted with an element through an [engagement + gesture](https://w3c.github.io/pointerlock/#dfn-engagement-gesture) (e.g. a + click or touch event) for the browser to allow this. \ No newline at end of file diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 09b6d1cc..672d8c98 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2610,6 +2610,27 @@ describe('Remote Frame Buffer Protocol Client', function () { client._canvas.dispatchEvent(ev); } + function supportsSendMouseMovementEvent() { + // Some browsers (like Safari) support the movementX / + // movementY properties of MouseEvent, but do not allow creation + // of non-trusted events with those properties. + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': 100, + 'movementY': 100 }); + return ev.movementX === 100 && ev.movementY === 100; + } + + function sendMouseMovementEvent(dx, dy) { + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': dx, + 'movementY': dy }); + client._canvas.dispatchEvent(ev); + } + function sendMouseButtonEvent(x, y, down, button) { let pos = elementToClient(x, y); let ev; @@ -2723,6 +2744,62 @@ describe('Remote Frame Buffer Protocol Client', function () { 50, 70, 0x0); }); + it('should ignore remote cursor position updates', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ }); + sendMouseMoveEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); + }); + + it('should handle remote mouse position updates in pointer lock mode', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const spy = sinon.spy(); + client.addEventListener("inputlock", spy); + let stub = sinon.stub(document, 'pointerLockElement'); + stub.get(function () { return client._canvas; }); + client._handlePointerLockChange(); + stub.restore(); + client._sock._websocket._receiveData(new Uint8Array([0x02, 0x02])); + expect(spy).to.have.been.calledOnce; + expect(spy.args[0][0].detail.pointer).to.be.true; + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ x: 50, y: 50 }); + sendMouseMovementEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 60, 60, 0x0); + }); + describe('Event Aggregation', function () { it('should send a single pointer event on mouse movement', function () { sendMouseMoveEvent(50, 70); diff --git a/vnc.html b/vnc.html index bbcf034f..75740a57 100644 --- a/vnc.html +++ b/vnc.html @@ -168,6 +168,11 @@ id="noVNC_fullscreen_button" class="noVNC_button noVNC_hidden" title="Fullscreen"> + + + Translate keyboard shurtcuts -
  • + +
    • @@ -227,6 +235,23 @@
    • +
  • +
    Keyboard Shortcuts
    +
    +
      +
    • + +
      • +
    • Ctrl+Shift+
      • +
    • 1 - Toggle Control Panel
      • +
    • 2 - Toggle Game Pointer Mode
      • +
    • 3 - Toggle Pointer Lock
      • +
    +
    +
    • +
  • Stream Quality