jcarr
/
noVNC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add UltraVNC touch gestures support

This commit is contained in:
Rui Reis 2024-05-28 14:41:15 +02:00
parent d80e3bfa2f
commit 2b738be8e6
5 changed files with 297 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/ui.js
View File

@ -180,6 +180,7 @@ const UI = {
UI.initSetting('shared', true);
UI.initSetting('view_only', false);
UI.initSetting('show_dot', false);
UI.initSetting('ultravnc_gestures', false);
UI.initSetting('path', 'websockify');
UI.initSetting('repeaterID', '');
UI.initSetting('reconnect', false);
@ -368,6 +369,7 @@ const UI = {
UI.addSettingChangeHandler('view_only', UI.updateViewOnly);
UI.addSettingChangeHandler('show_dot');
UI.addSettingChangeHandler('show_dot', UI.updateShowDotCursor);
UI.addSettingChangeHandler('ultravnc_gestures');
UI.addSettingChangeHandler('host');
UI.addSettingChangeHandler('port');
UI.addSettingChangeHandler('path');
@ -438,6 +440,7 @@ const UI = {
UI.disableSetting('port');
UI.disableSetting('path');
UI.disableSetting('repeaterID');
UI.disableSetting('ultravnc_gestures');
// Hide the controlbar after 2 seconds
UI.closeControlbarTimeout = setTimeout(UI.closeControlbar, 2000);
@ -448,6 +451,7 @@ const UI = {
UI.enableSetting('port');
UI.enableSetting('path');
UI.enableSetting('repeaterID');
UI.enableSetting('ultravnc_gestures');
UI.updatePowerButton();
UI.keepControlbar();
}
@ -1045,7 +1049,8 @@ const UI = {
UI.rfb = new RFB(document.getElementById('noVNC_container'), url,
{ shared: UI.getSetting('shared'),
repeaterID: UI.getSetting('repeaterID'),
credentials: { password: password } });
credentials: { password: password },
useUltraVNCGestures: UI.getSetting('ultravnc_gestures') });
} catch (exc) {
Log.Error("Failed to connect to server: " + exc);
UI.updateVisualState('disconnected');

1
core/encodings.js
View File

@ -23,6 +23,7 @@ export const encodings = {
pseudoEncodingCursor: -239,
pseudoEncodingQEMUExtendedKeyEvent: -258,
pseudoEncodingQEMULedEvent: -261,
pseudoEncodingGii: -305,
pseudoEncodingDesktopName: -307,
pseudoEncodingExtendedDesktopSize: -308,
pseudoEncodingXvp: -309,

112
core/input/touchhandlerultravnc.js Normal file
View File

@ -0,0 +1,112 @@
import * as Log from '../util/logging.js';
export default class TouchHandlerUltraVNC {
static PF_flag = 0x80000000; // Pressed Flag : active if the touch event is pressed, inactive if it's being released.
static R1_flag = 0x40000000; // Reserved 1
static IF_flag = 0x20000000; // Primary Flag : active if the touch event is the primary touch event.
static S1_flag = 0x10000000; // Size Flag : active if the message contains information about the size of the touch event. The events are currently all sent as symetrical ellipses.
static S2_flag = 0x8000000; // Reserved for asymetrical ellipses. Not supported yet and should be 0.
static RT_flag = 0x4000000; // Rectangle : the touch event is a rectangle instead of an ellipse.
static PR_flag = 0x2000000; // Pressure Flag : pressure of the touch. Currently unused.
static TI_flag = 0x1000000; // Timestamp : the timestamp of the touch event.
static HC_flag = 0x800000; // High Performance Counter
static LENGTH_16_flag = 0x10; // 16 bits signed for x touch coordinate followed by 16 bits signed for y together in a 32 bits word
static IDFORMAT_32 = 0x1; // 32 bits ID
static IDFORMAT_CLEAR = 0xF; // No more touch points
constructor() {
this._target = null;
this._currentTouches = [];
this._sendTouchesIntervalId = -1;
this._giiDeviceOrigin = 0;
this._isUltraVNCTouchActivated = false;
this._boundEventHandler = this._handleTouch.bind(this);
}
attach(target) {
this.detach();
this._target = target;
this._target.addEventListener('touchstart',
this._boundEventHandler);
this._target.addEventListener('touchmove',
this._boundEventHandler);
this._target.addEventListener('touchend',
this._boundEventHandler);
this._target.addEventListener('touchcancel',
this._boundEventHandler);
}
detach() {
if (!this._target) {
return;
}
this._target.removeEventListener('touchstart',
this._boundEventHandler);
this._target.removeEventListener('touchmove',
this._boundEventHandler);
this._target.removeEventListener('touchend',
this._boundEventHandler);
this._target.removeEventListener('touchcancel',
this._boundEventHandler);
clearInterval(this._sendTouchesIntervalId);
this._sendTouchesIntervalId = -1;
this._target = null;
}
_handleTouch(ev) {
Log.Debug("Gesture: " + ev.type);
if (!this._isUltraVNCTouchActivated) {
return;
}
if (ev.type === "touchstart") {
for (let i = 0; i < ev.changedTouches.length; i++) {
this._currentTouches.push({ event: ev.changedTouches[i], status: "POINTER_DOWN" });
}
if (this._sendTouchesIntervalId === -1 && this._target) {
this._dispatchTouchEvent(ev);
this._sendTouchesIntervalId = setInterval(() => {
this._dispatchTouchEvent(ev);
}, 200);
}
} else if (ev.type === "touchmove") {
for (let i = 0; i < ev.changedTouches.length; i++) {
const index = this._currentTouches.findIndex(t => t.event.identifier === ev.changedTouches[i].identifier);
if (index !== -1) {
this._currentTouches[index].event = ev.changedTouches[i];
this._currentTouches[index].status = "POINTER_UPDATE";
}
}
} else if (ev.type === "touchend" || ev.type === "touchcancel") {
for (let i = 0; i < ev.changedTouches.length; i++) {
const index = this._currentTouches.findIndex(t => t.event.identifier === ev.changedTouches[i].identifier);
if (index !== -1) {
this._currentTouches[index].status = "POINTER_UP";
}
}
}
}
_dispatchTouchEvent(ev) {
let tev = new CustomEvent('ultravnctouch', { event: ev, detail: { currentTouches: this._currentTouches, giiDeviceOrigin: this._giiDeviceOrigin } });
this._target.dispatchEvent(tev);
}
_removeTouch(index) {
this._currentTouches.splice(index, 1);
}
_interruptTouches() {
clearInterval(this._sendTouchesIntervalId);
this._sendTouchesIntervalId = -1;
}
}

184
core/rfb.js
View File

@ -19,6 +19,7 @@ import Inflator from "./inflator.js";
import Deflator from "./deflator.js";
import Keyboard from "./input/keyboard.js";
import GestureHandler from "./input/gesturehandler.js";
import TouchHandlerUltraVNC from "./input/touchhandlerultravnc.js";
import Cursor from "./util/cursor.js";
import Websock from "./websock.js";
import KeyTable from "./input/keysym.js";
@ -117,6 +118,7 @@ export default class RFB extends EventTargetMixin {
this._shared = 'shared' in options ? !!options.shared : true;
this._repeaterID = options.repeaterID || '';
this._wsProtocols = options.wsProtocols || [];
this._useUltraVNCGestures = options.useUltraVNCGestures || false;
// Internal state
this._rfbConnectionState = '';
@ -202,6 +204,7 @@ export default class RFB extends EventTargetMixin {
handleMouse: this._handleMouse.bind(this),
handleWheel: this._handleWheel.bind(this),
handleGesture: this._handleGesture.bind(this),
handleUltraVNCTouch: this._handleUltraVNCTouch.bind(this),
handleRSAAESCredentialsRequired: this._handleRSAAESCredentialsRequired.bind(this),
handleRSAAESServerVerification: this._handleRSAAESServerVerification.bind(this),
};
@ -263,7 +266,11 @@ export default class RFB extends EventTargetMixin {
this._remoteCapsLock = null; // Null indicates unknown or irrelevant
this._remoteNumLock = null;
this._gestures = new GestureHandler();
if (this._useUltraVNCGestures) {
this._gestures = new TouchHandlerUltraVNC();
} else {
this._gestures = new GestureHandler();
}
this._sock = new Websock();
this._sock.on('open', this._socketOpen.bind(this));
@ -594,9 +601,13 @@ export default class RFB extends EventTargetMixin {
this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel);
// Gesture events
this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture);
this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
if (this._useUltraVNCGestures) {
this._canvas.addEventListener('ultravnctouch', this._eventHandlers.handleUltraVNCTouch);
} else {
this._canvas.addEventListener('gesturestart', this._eventHandlers.handleGesture);
this._canvas.addEventListener('gesturemove', this._eventHandlers.handleGesture);
this._canvas.addEventListener('gestureend', this._eventHandlers.handleGesture);
}
Log.Debug("<< RFB.connect");
}
@ -604,9 +615,13 @@ export default class RFB extends EventTargetMixin {
_disconnect() {
Log.Debug(">> RFB.disconnect");
this._cursor.detach();
this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture);
this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture);
this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture);
if (this._useUltraVNCGestures) {
this._canvas.removeEventListener('ultravnctouch', this._eventHandlers.handleUltraVNCTouch);
} else {
this._canvas.removeEventListener('gesturestart', this._eventHandlers.handleGesture);
this._canvas.removeEventListener('gesturemove', this._eventHandlers.handleGesture);
this._canvas.removeEventListener('gestureend', this._eventHandlers.handleGesture);
}
this._canvas.removeEventListener("wheel", this._eventHandlers.handleWheel);
this._canvas.removeEventListener('mousedown', this._eventHandlers.handleMouse);
this._canvas.removeEventListener('mouseup', this._eventHandlers.handleMouse);
@ -1374,6 +1389,87 @@ export default class RFB extends EventTargetMixin {
}
}
_handleUltraVNCTouch(ev) {
Log.Debug("SENDING " + ev.detail.currentTouches.length + " TOUCH(ES)");
this._sock.sQpush8(253); // GII message type
this._sock.sQpush8(128); // GII event
this._sock.sQpush16(4 + 16 + (6 * 2 * ev.detail.currentTouches.length)); // Length
this._sock.sQpush8(4 + 16 + (6 * 2 * ev.detail.currentTouches.length)); // eventSize
this._sock.sQpush8(12); // eventType
this._sock.sQpush16(0); // padding
this._sock.sQpush32(ev.detail.giiDeviceOrigin); // deviceOrigin
this._sock.sQpush32(ev.detail.currentTouches.length); // first
this._sock.sQpush32(6 * ev.detail.currentTouches.length); // count
let pointerUpIds = [];
// Send all current touches
for (let i = 0; i < ev.detail.currentTouches.length; i++) {
Log.Debug("Touch Id: " + ev.detail.currentTouches[i].event.identifier);
let valuatorFlag = 0x00000000;
valuatorFlag |= TouchHandlerUltraVNC.LENGTH_16_flag;
valuatorFlag |= TouchHandlerUltraVNC.IDFORMAT_32;
if (ev.detail.currentTouches[i].status !== "POINTER_UP") valuatorFlag |= TouchHandlerUltraVNC.PF_flag;
if (ev.detail.currentTouches[i].event.identifier === 0) valuatorFlag |= TouchHandlerUltraVNC.IF_flag; // IF_flag
this._sock.sQpush32(valuatorFlag);
this._sock.sQpush32(ev.detail.currentTouches[i].event.identifier);
let scaledPosition = clientToElement(ev.detail.currentTouches[i].event.clientX, ev.detail.currentTouches[i].event.clientY,
this._canvas);
if ((valuatorFlag & TouchHandlerUltraVNC.LENGTH_16_flag) !== 0) {
let scaledX16 = Math.floor(scaledPosition.x) & 0xFFFF;
let scaledY16 = Math.floor(scaledPosition.y) & 0xFFFF;
let coordinates = (Math.floor(scaledX16) << 16) | (Math.floor(scaledY16));
this._sock.sQpush32(coordinates);
}
// Keep track of last released touches
if (ev.detail.currentTouches[i].status === "POINTER_UP") {
pointerUpIds.push(ev.detail.currentTouches[i].event.identifier);
}
}
this._sock.flush();
// Remove released touches from current touches in handler
for (let i = 0; i < pointerUpIds.length; i++) {
const index = ev.detail.currentTouches.findIndex(t => t.event.identifier === pointerUpIds[i]);
if (index !== -1) {
this._gestures._removeTouch(index);
}
}
// Interrupt touch sending interval
if (ev.detail.currentTouches.length === 0 && this._sendTouchesIntervalId !== -1) {
Log.Debug("NO MORE TOUCHES\n");
this._gestures._interruptTouches();
this._sendEmptyTouch(ev.detail.giiDeviceOrigin);
return;
}
}
_sendEmptyTouch(giiDeviceOrigin) {
let valuatorFlag = 0x00000000;
valuatorFlag |= TouchHandlerUltraVNC.LENGTH_16_flag;
valuatorFlag |= TouchHandlerUltraVNC.IDFORMAT_CLEAR;
this._sock.sQpush8(253); // GII message type
this._sock.sQpush8(128); // GII event
this._sock.sQpush16(24); // Header length
this._sock.sQpush8(24); // Event size
this._sock.sQpush8(12); // eventType
this._sock.sQpush16(0); // padding
this._sock.sQpush32(giiDeviceOrigin); // deviceOrigin
this._sock.sQpush32(1); // first
this._sock.sQpush32(4); // Count
this._sock.sQpush32(valuatorFlag); // Flag
this._sock.sQpush32(0); // Empty Id
this._sock.sQpush32(0); // Empty coordinates
this._sock.flush();
}
// Message Handlers
_negotiateProtocolVersion() {
@ -2139,6 +2235,10 @@ export default class RFB extends EventTargetMixin {
encs.push(encodings.pseudoEncodingDesktopName);
encs.push(encodings.pseudoEncodingExtendedClipboard);
if (this._useUltraVNCGestures) {
encs.push(encodings.pseudoEncodingGii);
}
if (this._fbDepth == 24) {
encs.push(encodings.pseudoEncodingVMwareCursor);
encs.push(encodings.pseudoEncodingCursor);
@ -2434,6 +2534,25 @@ export default class RFB extends EventTargetMixin {
return true;
}
_handleGiiMsg() {
let giiMsgSubtype = this._sock.rQshift8();
switch (giiMsgSubtype) {
case 129: // GII Version Message
this._sock.rQskipBytes(34);
RFB.messages.giiVersionMessage(this._sock);
RFB.messages.giiDeviceCreation(this._sock);
break;
case 130: // GII Device Creation
this._sock.rQshiftBytes(2);
this._gestures._giiDeviceOrigin = this._sock.rQshift32();
if (this._gestures._giiDeviceOrigin) this._gestures._isUltraVNCTouchActivated = true;
break;
}
return true;
}
_normalMsg() {
let msgType;
if (this._FBU.rects > 0) {
@ -2485,6 +2604,9 @@ export default class RFB extends EventTargetMixin {
case 250: // XVP
return this._handleXvpMsg();
case 253: // GII
return this._handleGiiMsg();
default:
this._fail("Unexpected server message (type " + msgType + ")");
Log.Debug("sock.rQpeekBytes(30): " + this._sock.rQpeekBytes(30));
@ -2933,6 +3055,16 @@ export default class RFB extends EventTargetMixin {
"raw", passwordChars, { name: "DES-ECB" }, false, ["encrypt"]);
return legacyCrypto.encrypt({ name: "DES-ECB" }, key, challenge);
}
static stringAsByteArrayWithPadding(str, size) {
let full = new Uint8Array(size);
let utf8Encode = new TextEncoder();
let strArray = utf8Encode.encode(str);
for (let i = 0; i < strArray.length; i++) {
full[i] = strArray[i];
}
return full;
}
}
// Class Methods
@ -3224,6 +3356,44 @@ RFB.messages = {
sock.sQpush8(ver);
sock.sQpush8(op);
sock.flush();
},
giiVersionMessage(sock) {
sock.sQpush8(253); // gii msg-type
sock.sQpush8(129); // gii version sub-msg-type
sock.sQpush16(2); // length
sock.sQpush16(1); // version
sock.flush();
},
giiDeviceCreation(sock) {
sock.sQpush8(253); // gii msg-type
sock.sQpush8(130); // gii device creation sub-msg-type
sock.sQpush16(172); // length
sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NOVNC-MT", 31)); // device name
sock.sQpush8(0); // DNTerm
sock.sQpush32(0x0908); // vendorID
sock.sQpush32(0x000b); // productID
sock.sQpush32(0x00002000); // eventMask
sock.sQpush32(0); // numRegisters
sock.sQpush32(1); // numValuators
sock.sQpush32(5); // numButtons
sock.sQpush32(0); // index
sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NOVNC Multitouch Device", 74)); // longName
sock.sQpush8(0); // LNTerm
sock.sQpushBytes(RFB.stringAsByteArrayWithPadding("NMD", 4)); // shortName
sock.sQpush8(0); // SNTerm
sock.sQpush32(0); // rangeMin
sock.sQpush32(0); // rangeCenter
sock.sQpush32(0); // rangeMax
sock.sQpush32(0); // SIUnit
sock.sQpush32(0); // SIAdd
sock.sQpush32(0); // SIMul
sock.sQpush32(0); // SIDiv
sock.sQpush32(0); // SIShift
sock.flush();
}
};

1
vnc.html
View File

@ -220,6 +220,7 @@
<li><hr></li>
<li>
<label><input id="noVNC_setting_show_dot" type="checkbox"> Show Dot when No Cursor</label>
<label><input id="noVNC_setting_ultravnc_gestures" type="checkbox"> Use UltraVNC gestures</label>
</li>
<li><hr></li>
<!-- Logging selection dropdown -->