From 663a198ad5a9384aa926698835c899345c4b4ac1 Mon Sep 17 00:00:00 2001 From: Rui Reis Date: Tue, 28 May 2024 14:41:15 +0200 Subject: [PATCH] Add UltraVNC touch gestures support --- app/ui.js | 7 +- core/encodings.js | 1 + core/input/touchhandlerultravnc.js | 112 ++++++++++++++++++ core/rfb.js | 184 +++++++++++++++++++++++++++-- vnc.html | 1 + 5 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 core/input/touchhandlerultravnc.js diff --git a/app/ui.js b/app/ui.js index fd23c800..adab59a8 100644 --- a/app/ui.js +++ b/app/ui.js @@ -185,6 +185,7 @@ const UI = { UI.initSetting('bell', 'on'); UI.initSetting('view_only', false); UI.initSetting('show_dot', false); + UI.initSetting('ultravnc_gestures', false); UI.initSetting('path', 'websockify'); UI.initSetting('repeaterID', ''); UI.initSetting('reconnect', false); @@ -371,6 +372,7 @@ const UI = { UI.addSettingChangeHandler('view_only', UI.updateViewOnly); UI.addSettingChangeHandler('show_dot'); UI.addSettingChangeHandler('show_dot', UI.updateShowDotCursor); + UI.addSettingChangeHandler('ultravnc_gestures'); UI.addSettingChangeHandler('host'); UI.addSettingChangeHandler('port'); UI.addSettingChangeHandler('path'); @@ -441,6 +443,7 @@ const UI = { UI.disableSetting('port'); UI.disableSetting('path'); UI.disableSetting('repeaterID'); + UI.disableSetting('ultravnc_gestures'); // Hide the controlbar after 2 seconds UI.closeControlbarTimeout = setTimeout(UI.closeControlbar, 2000); @@ -451,6 +454,7 @@ const UI = { UI.enableSetting('port'); UI.enableSetting('path'); UI.enableSetting('repeaterID'); + UI.enableSetting('ultravnc_gestures'); UI.updatePowerButton(); UI.keepControlbar(); } @@ -1072,7 +1076,8 @@ const UI = { url.href, { shared: UI.getSetting('shared'), repeaterID: UI.getSetting('repeaterID'), - credentials: { password: password } }); + credentials: { password: password }, + useUltraVNCGestures: UI.getSetting('ultravnc_gestures') }); } catch (exc) { Log.Error("Failed to connect to server: " + exc); UI.updateVisualState('disconnected'); diff --git a/core/encodings.js b/core/encodings.js index bf25ac91..27a635da 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -25,6 +25,7 @@ export const encodings = { pseudoEncodingCursor: -239, pseudoEncodingQEMUExtendedKeyEvent: -258, pseudoEncodingQEMULedEvent: -261, + pseudoEncodingGii: -305, pseudoEncodingDesktopName: -307, pseudoEncodingExtendedDesktopSize: -308, pseudoEncodingXvp: -309, diff --git a/core/input/touchhandlerultravnc.js b/core/input/touchhandlerultravnc.js new file mode 100644 index 00000000..c9ccac91 --- /dev/null +++ b/core/input/touchhandlerultravnc.js @@ -0,0 +1,112 @@ +import * as Log from '../util/logging.js'; + +export default class TouchHandlerUltraVNC { + static PF_flag = 0x80000000; // Pressed Flag : active if the touch event is pressed, inactive if it's being released. + static R1_flag = 0x40000000; // Reserved 1 + static IF_flag = 0x20000000; // Primary Flag : active if the touch event is the primary touch event. + static S1_flag = 0x10000000; // Size Flag : active if the message contains information about the size of the touch event. The events are currently all sent as symetrical ellipses. + static S2_flag = 0x8000000; // Reserved for asymetrical ellipses. Not supported yet and should be 0. + static RT_flag = 0x4000000; // Rectangle : the touch event is a rectangle instead of an ellipse. + static PR_flag = 0x2000000; // Pressure Flag : pressure of the touch. Currently unused. + static TI_flag = 0x1000000; // Timestamp : the timestamp of the touch event. + static HC_flag = 0x800000; // High Performance Counter + + static LENGTH_16_flag = 0x10; // 16 bits signed for x touch coordinate followed by 16 bits signed for y together in a 32 bits word + static IDFORMAT_32 = 0x1; // 32 bits ID + static IDFORMAT_CLEAR = 0xF; // No more touch points + + constructor() { + this._target = null; + + this._currentTouches = []; + this._sendTouchesIntervalId = -1; + this._giiDeviceOrigin = 0; + this._isUltraVNCTouchActivated = false; + + this._boundEventHandler = this._handleTouch.bind(this); + } + + attach(target) { + this.detach(); + + this._target = target; + this._target.addEventListener('touchstart', + this._boundEventHandler); + this._target.addEventListener('touchmove', + this._boundEventHandler); + this._target.addEventListener('touchend', + this._boundEventHandler); + this._target.addEventListener('touchcancel', + this._boundEventHandler); + } + + detach() { + if (!this._target) { + return; + } + + this._target.removeEventListener('touchstart', + this._boundEventHandler); + this._target.removeEventListener('touchmove', + this._boundEventHandler); + this._target.removeEventListener('touchend', + this._boundEventHandler); + this._target.removeEventListener('touchcancel', + this._boundEventHandler); + + clearInterval(this._sendTouchesIntervalId); + this._sendTouchesIntervalId = -1; + + this._target = null; + } + + _handleTouch(ev) { + Log.Debug("Gesture: " + ev.type); + + if (!this._isUltraVNCTouchActivated) { + return; + } + + if (ev.type === "touchstart") { + for (let i = 0; i < ev.changedTouches.length; i++) { + this._currentTouches.push({ event: ev.changedTouches[i], status: "POINTER_DOWN" }); + } + + if (this._sendTouchesIntervalId === -1 && this._target) { + this._dispatchTouchEvent(ev); + this._sendTouchesIntervalId = setInterval(() => { + this._dispatchTouchEvent(ev); + }, 200); + } + } else if (ev.type === "touchmove") { + for (let i = 0; i < ev.changedTouches.length; i++) { + const index = this._currentTouches.findIndex(t => t.event.identifier === ev.changedTouches[i].identifier); + if (index !== -1) { + this._currentTouches[index].event = ev.changedTouches[i]; + this._currentTouches[index].status = "POINTER_UPDATE"; + } + } + } else if (ev.type === "touchend" || ev.type === "touchcancel") { + for (let i = 0; i < ev.changedTouches.length; i++) { + const index = this._currentTouches.findIndex(t => t.event.identifier === ev.changedTouches[i].identifier); + if (index !== -1) { + this._currentTouches[index].status = "POINTER_UP"; + } + } + } + } + + _dispatchTouchEvent(ev) { + let tev = new CustomEvent('ultravnctouch', { event: ev, detail: { currentTouches: this._currentTouches, giiDeviceOrigin: this._giiDeviceOrigin } }); + this._target.dispatchEvent(tev); + } + + _removeTouch(index) { + this._currentTouches.splice(index, 1); + } + + _interruptTouches() { + clearInterval(this._sendTouchesIntervalId); + this._sendTouchesIntervalId = -1; + } +} \ No newline at end of file diff --git a/core/rfb.js b/core/rfb.js index 9559e487..995ab798 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -19,6 +19,7 @@ import Inflator from "./inflator.js"; import Deflator from "./deflator.js"; import Keyboard from "./input/keyboard.js"; import GestureHandler from "./input/gesturehandler.js"; +import TouchHandlerUltraVNC from "./input/touchhandlerultravnc.js"; import Cursor from "./util/cursor.js"; import Websock from "./websock.js"; import KeyTable from "./input/keysym.js"; @@ -119,6 +120,7 @@ export default class RFB extends EventTargetMixin { this._shared = 'shared' in options ? !!options.shared : true; this._repeaterID = options.repeaterID || ''; this._wsProtocols = options.wsProtocols || []; + this._useUltraVNCGestures = options.useUltraVNCGestures || false; // Internal state this._rfbConnectionState = ''; @@ -204,6 +206,7 @@ export default class RFB extends EventTargetMixin { handleMouse: this._handleMouse.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), + handleUltraVNCTouch: this._handleUltraVNCTouch.bind(this), handleRSAAESCredentialsRequired: this._handleRSAAESCredentialsRequired.bind(this), handleRSAAESServerVerification: this._handleRSAAESServerVerification.bind(this), }; @@ -267,7 +270,11 @@ export default class RFB extends EventTargetMixin { this._remoteCapsLock = null; // Null indicates unknown or irrelevant this._remoteNumLock = null; - this._gestures = new GestureHandler(); + if (this._useUltraVNCGestures) { + this._gestures = new TouchHandlerUltraVNC(); + } else { + this._gestures = new GestureHandler(); + } this._sock = new Websock(); this._sock.on('open', this._socketOpen.bind(this)); @@ -598,9 +605,13 @@ export default class RFB extends EventTargetMixin { this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); // Gesture events - this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture); - this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture); - this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture); + if (this._useUltraVNCGestures) { + this._canvas.addEventListener('ultravnctouch', this._eventHandlers.handleUltraVNCTouch); + } else { + this._canvas.addEventListener('gesturestart', this._eventHandlers.handleGesture); + this._canvas.addEventListener('gesturemove', this._eventHandlers.handleGesture); + this._canvas.addEventListener('gestureend', this._eventHandlers.handleGesture); + } Log.Debug("<< RFB.connect"); } @@ -608,9 +619,13 @@ export default class RFB extends EventTargetMixin { _disconnect() { Log.Debug(">> RFB.disconnect"); this._cursor.detach(); - this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture); - this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture); - this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture); + if (this._useUltraVNCGestures) { + this._canvas.removeEventListener('ultravnctouch', this._eventHandlers.handleUltraVNCTouch); + } else { + this._canvas.removeEventListener('gesturestart', this._eventHandlers.handleGesture); + this._canvas.removeEventListener('gesturemove', this._eventHandlers.handleGesture); + this._canvas.removeEventListener('gestureend', this._eventHandlers.handleGesture); + } this._canvas.removeEventListener("wheel", this._eventHandlers.handleWheel); this._canvas.removeEventListener('mousedown', this._eventHandlers.handleMouse); this._canvas.removeEventListener('mouseup', this._eventHandlers.handleMouse); @@ -1378,6 +1393,87 @@ export default class RFB extends EventTargetMixin { } } + _handleUltraVNCTouch(ev) { + Log.Debug("SENDING " + ev.detail.currentTouches.length + " TOUCH(ES)"); + this._sock.sQpush8(253); // GII message type + this._sock.sQpush8(128); // GII event + this._sock.sQpush16(4 + 16 + (6 * 2 * ev.detail.currentTouches.length)); // Length + this._sock.sQpush8(4 + 16 + (6 * 2 * ev.detail.currentTouches.length)); // eventSize + this._sock.sQpush8(12); // eventType + this._sock.sQpush16(0); // padding + this._sock.sQpush32(ev.detail.giiDeviceOrigin); // deviceOrigin + this._sock.sQpush32(ev.detail.currentTouches.length); // first + this._sock.sQpush32(6 * ev.detail.currentTouches.length); // count + + let pointerUpIds = []; + + // Send all current touches + for (let i = 0; i < ev.detail.currentTouches.length; i++) { + Log.Debug("Touch Id: " + ev.detail.currentTouches[i].event.identifier); + let valuatorFlag = 0x00000000; + valuatorFlag |= TouchHandlerUltraVNC.LENGTH_16_flag; + valuatorFlag |= TouchHandlerUltraVNC.IDFORMAT_32; + if (ev.detail.currentTouches[i].status !== "POINTER_UP") valuatorFlag |= TouchHandlerUltraVNC.PF_flag; + if (ev.detail.currentTouches[i].event.identifier === 0) valuatorFlag |= TouchHandlerUltraVNC.IF_flag; // IF_flag + + this._sock.sQpush32(valuatorFlag); + this._sock.sQpush32(ev.detail.currentTouches[i].event.identifier); + + let scaledPosition = clientToElement(ev.detail.currentTouches[i].event.clientX, ev.detail.currentTouches[i].event.clientY, + this._canvas); + + if ((valuatorFlag & TouchHandlerUltraVNC.LENGTH_16_flag) !== 0) { + let scaledX16 = Math.floor(scaledPosition.x) & 0xFFFF; + let scaledY16 = Math.floor(scaledPosition.y) & 0xFFFF; + let coordinates = (Math.floor(scaledX16) << 16) | (Math.floor(scaledY16)); + this._sock.sQpush32(coordinates); + } + + // Keep track of last released touches + if (ev.detail.currentTouches[i].status === "POINTER_UP") { + pointerUpIds.push(ev.detail.currentTouches[i].event.identifier); + } + } + + this._sock.flush(); + + // Remove released touches from current touches in handler + for (let i = 0; i < pointerUpIds.length; i++) { + const index = ev.detail.currentTouches.findIndex(t => t.event.identifier === pointerUpIds[i]); + if (index !== -1) { + this._gestures._removeTouch(index); + } + } + + // Interrupt touch sending interval + if (ev.detail.currentTouches.length === 0 && this._sendTouchesIntervalId !== -1) { + Log.Debug("NO MORE TOUCHES\n"); + this._gestures._interruptTouches(); + this._sendEmptyTouch(ev.detail.giiDeviceOrigin); + return; + } + } + + _sendEmptyTouch(giiDeviceOrigin) { + let valuatorFlag = 0x00000000; + valuatorFlag |= TouchHandlerUltraVNC.LENGTH_16_flag; + valuatorFlag |= TouchHandlerUltraVNC.IDFORMAT_CLEAR; + + this._sock.sQpush8(253); // GII message type + this._sock.sQpush8(128); // GII event + this._sock.sQpush16(24); // Header length + this._sock.sQpush8(24); // Event size + this._sock.sQpush8(12); // eventType + this._sock.sQpush16(0); // padding + this._sock.sQpush32(giiDeviceOrigin); // deviceOrigin + this._sock.sQpush32(1); // first + this._sock.sQpush32(4); // Count + this._sock.sQpush32(valuatorFlag); // Flag + this._sock.sQpush32(0); // Empty Id + this._sock.sQpush32(0); // Empty coordinates + this._sock.flush(); + } + // Message handlers _negotiateProtocolVersion() { @@ -2147,6 +2243,10 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingDesktopName); encs.push(encodings.pseudoEncodingExtendedClipboard); + if (this._useUltraVNCGestures) { + encs.push(encodings.pseudoEncodingGii); + } + if (this._fbDepth == 24) { encs.push(encodings.pseudoEncodingVMwareCursor); encs.push(encodings.pseudoEncodingCursor); @@ -2442,6 +2542,25 @@ export default class RFB extends EventTargetMixin { return true; } + _handleGiiMsg() { + let giiMsgSubtype = this._sock.rQshift8(); + + switch (giiMsgSubtype) { + case 129: // GII Version Message + this._sock.rQskipBytes(34); + RFB.messages.giiVersionMessage(this._sock); + RFB.messages.giiDeviceCreation(this._sock); + break; + case 130: // GII Device Creation + this._sock.rQshiftBytes(2); + this._gestures._giiDeviceOrigin = this._sock.rQshift32(); + if (this._gestures._giiDeviceOrigin) this._gestures._isUltraVNCTouchActivated = true; + break; + } + + return true; + } + _normalMsg() { let msgType; if (this._FBU.rects > 0) { @@ -2493,6 +2612,9 @@ export default class RFB extends EventTargetMixin { case 250: // XVP return this._handleXvpMsg(); + case 253: // GII + return this._handleGiiMsg(); + default: this._fail("Unexpected server message (type " + msgType + ")"); Log.Debug("sock.rQpeekBytes(30): " + this._sock.rQpeekBytes(30)); @@ -2941,6 +3063,16 @@ export default class RFB extends EventTargetMixin { "raw", passwordChars, { name: "DES-ECB" }, false, ["encrypt"]); return legacyCrypto.encrypt({ name: "DES-ECB" }, key, challenge); } + + static stringAsByteArrayWithPadding(str, size) { + let full = new Uint8Array(size); + let utf8Encode = new TextEncoder(); + let strArray = utf8Encode.encode(str); + for (let i = 0; i < strArray.length; i++) { + full[i] = strArray[i]; + } + return full; + } } // Class Methods @@ -3232,6 +3364,44 @@ RFB.messages = { sock.sQpush8(ver); sock.sQpush8(op); + sock.flush(); + }, + + giiVersionMessage(sock) { + sock.sQpush8(253); // gii msg-type + sock.sQpush8(129); // gii version sub-msg-type + sock.sQpush16(2); // length + sock.sQpush16(1); // version + + sock.flush(); + }, + + giiDeviceCreation(sock) { + sock.sQpush8(253); // gii msg-type + sock.sQpush8(130); // gii device creation sub-msg-type + sock.sQpush16(172); // length + sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NOVNC-MT", 31)); // device name + sock.sQpush8(0); // DNTerm + sock.sQpush32(0x0908); // vendorID + sock.sQpush32(0x000b); // productID + sock.sQpush32(0x00002000); // eventMask + sock.sQpush32(0); // numRegisters + sock.sQpush32(1); // numValuators + sock.sQpush32(5); // numButtons + sock.sQpush32(0); // index + sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NOVNC Multitouch Device", 74)); // longName + sock.sQpush8(0); // LNTerm + sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NMD", 4)); // shortName + sock.sQpush8(0); // SNTerm + sock.sQpush32(0); // rangeMin + sock.sQpush32(0); // rangeCenter + sock.sQpush32(0); // rangeMax + sock.sQpush32(0); // SIUnit + sock.sQpush32(0); // SIAdd + sock.sQpush32(0); // SIMul + sock.sQpush32(0); // SIDiv + sock.sQpush32(0); // SIShift + sock.flush(); } }; diff --git a/vnc.html b/vnc.html index c2cc4e55..44c43718 100644 --- a/vnc.html +++ b/vnc.html @@ -270,6 +270,7 @@
  • +