From 0a6aec357850e4681ecfc6cbf74196dc96dbf75b Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 8 Jun 2020 15:35:28 +0200 Subject: [PATCH 01/11] Avoid printing the Websock buffer in tests It takes forever and just messes up the output. --- tests/test.rfb.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index d5fa4733..37ef29d6 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -107,10 +107,12 @@ describe('Remote Frame Buffer Protocol Client', function () { this._rQ = rQ; }; + // Avoiding printing the entire Websock buffer on errors + Websock.prototype.toString = function () { return "[object Websock]"; }; }); after(function () { - Websock.prototype._allocateBuffers = Websock.prototype._oldAllocateBuffers; + delete Websock.prototype.toString; this.clock.restore(); window.requestAnimationFrame = raf; }); From 97b86abc948f3f1d5d20864bb4325a0feb6f34bb Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 11 Jun 2020 16:31:09 +0200 Subject: [PATCH 02/11] Avoid fractional pixel sizes from Display --- core/display.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/display.js b/core/display.js index 7b7f5363..cf1a51aa 100644 --- a/core/display.js +++ b/core/display.js @@ -9,6 +9,7 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; import { supportsImageMetadata } from './util/browser.js'; +import { toSigned32bit } from './util/int.js'; export default class Display { constructor(target) { @@ -190,14 +191,14 @@ export default class Display { if (this._scale === 0) { return 0; } - return x / this._scale + this._viewportLoc.x; + return toSigned32bit(x / this._scale + this._viewportLoc.x); } absY(y) { if (this._scale === 0) { return 0; } - return y / this._scale + this._viewportLoc.y; + return toSigned32bit(y / this._scale + this._viewportLoc.y); } resize(width, height) { From 440ec8a0b696640ce1276b0359ed573762dce245 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Thu, 11 Jun 2020 16:43:38 +0200 Subject: [PATCH 03/11] Start fake test clock at real clock time Some code relies on the clock having a somewhat sane value, so let's not start at 0. --- tests/test.rfb.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 37ef29d6..62efefdb 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -91,7 +91,7 @@ describe('Remote Frame Buffer Protocol Client', function () { after(FakeWebSocket.restore); before(function () { - this.clock = clock = sinon.useFakeTimers(); + this.clock = clock = sinon.useFakeTimers(Date.now()); // sinon doesn't support this yet raf = window.requestAnimationFrame; window.requestAnimationFrame = setTimeout; From 8be924c9d96324800ac38ea086e500ee71255d0d Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Mon, 18 May 2020 15:17:06 +0200 Subject: [PATCH 04/11] Add touch gestures for mouse emulation Add several single and multitouch gestures to simulate various mouse actions that would otherwise be impossible to perform. This replaces the old system where you could select which mouse button a single touch would generate. --- app/images/mouse_left.svg | 92 --- app/images/mouse_middle.svg | 92 --- app/images/mouse_none.svg | 92 --- app/images/mouse_right.svg | 92 --- app/ui.js | 31 - core/input/gesturehandler.js | 567 +++++++++++++++++++ core/input/mouse.js | 58 +- core/rfb.js | 183 +++++- core/util/element.js | 32 ++ docs/API.md | 6 - tests/test.gesturehandler.js | 1031 ++++++++++++++++++++++++++++++++++ tests/test.mouse.js | 126 ----- tests/test.rfb.js | 750 ++++++++++++++++++++++++- vnc.html | 12 - 14 files changed, 2562 insertions(+), 602 deletions(-) delete mode 100644 app/images/mouse_left.svg delete mode 100644 app/images/mouse_middle.svg delete mode 100644 app/images/mouse_none.svg delete mode 100644 app/images/mouse_right.svg create mode 100644 core/input/gesturehandler.js create mode 100644 core/util/element.js create mode 100644 tests/test.gesturehandler.js diff --git a/app/images/mouse_left.svg b/app/images/mouse_left.svg deleted file mode 100644 index ce4cca41..00000000 --- a/app/images/mouse_left.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - diff --git a/app/images/mouse_middle.svg b/app/images/mouse_middle.svg deleted file mode 100644 index 6603425c..00000000 --- a/app/images/mouse_middle.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - diff --git a/app/images/mouse_none.svg b/app/images/mouse_none.svg deleted file mode 100644 index 3e0f838a..00000000 --- a/app/images/mouse_none.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - diff --git a/app/images/mouse_right.svg b/app/images/mouse_right.svg deleted file mode 100644 index f4bad767..00000000 --- a/app/images/mouse_right.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - diff --git a/app/ui.js b/app/ui.js index a039e94b..c70743dc 100644 --- a/app/ui.js +++ b/app/ui.js @@ -234,14 +234,6 @@ const UI = { }, addTouchSpecificHandlers() { - document.getElementById("noVNC_mouse_button0") - .addEventListener('click', () => UI.setMouseButton(1)); - document.getElementById("noVNC_mouse_button1") - .addEventListener('click', () => UI.setMouseButton(2)); - document.getElementById("noVNC_mouse_button2") - .addEventListener('click', () => UI.setMouseButton(4)); - document.getElementById("noVNC_mouse_button4") - .addEventListener('click', () => UI.setMouseButton(0)); document.getElementById("noVNC_keyboard_button") .addEventListener('click', UI.toggleVirtualKeyboard); @@ -430,7 +422,6 @@ const UI = { UI.disableSetting('port'); UI.disableSetting('path'); UI.disableSetting('repeaterID'); - UI.setMouseButton(1); // Hide the controlbar after 2 seconds UI.closeControlbarTimeout = setTimeout(UI.closeControlbar, 2000); @@ -1633,24 +1624,6 @@ const UI = { * MISC * ------v------*/ - setMouseButton(num) { - const viewOnly = UI.rfb.viewOnly; - if (UI.rfb && !viewOnly) { - UI.rfb.touchButton = num; - } - - const blist = [0, 1, 2, 4]; - for (let b = 0; b < blist.length; b++) { - const button = document.getElementById('noVNC_mouse_button' + - blist[b]); - if (blist[b] === num && !viewOnly) { - button.classList.remove("noVNC_hidden"); - } else { - button.classList.add("noVNC_hidden"); - } - } - }, - updateViewOnly() { if (!UI.rfb) return; UI.rfb.viewOnly = UI.getSetting('view_only'); @@ -1661,8 +1634,6 @@ const UI = { .classList.add('noVNC_hidden'); document.getElementById('noVNC_toggle_extra_keys_button') .classList.add('noVNC_hidden'); - document.getElementById('noVNC_mouse_button' + UI.rfb.touchButton) - .classList.add('noVNC_hidden'); document.getElementById('noVNC_clipboard_button') .classList.add('noVNC_hidden'); } else { @@ -1670,8 +1641,6 @@ const UI = { .classList.remove('noVNC_hidden'); document.getElementById('noVNC_toggle_extra_keys_button') .classList.remove('noVNC_hidden'); - document.getElementById('noVNC_mouse_button' + UI.rfb.touchButton) - .classList.remove('noVNC_hidden'); document.getElementById('noVNC_clipboard_button') .classList.remove('noVNC_hidden'); } diff --git a/core/input/gesturehandler.js b/core/input/gesturehandler.js new file mode 100644 index 00000000..6fa72d2a --- /dev/null +++ b/core/input/gesturehandler.js @@ -0,0 +1,567 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2020 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +const GH_NOGESTURE = 0; +const GH_ONETAP = 1; +const GH_TWOTAP = 2; +const GH_THREETAP = 4; +const GH_DRAG = 8; +const GH_LONGPRESS = 16; +const GH_TWODRAG = 32; +const GH_PINCH = 64; + +const GH_INITSTATE = 127; + +const GH_MOVE_THRESHOLD = 50; +const GH_ANGLE_THRESHOLD = 90; // Degrees + +// Timeout when waiting for gestures (ms) +const GH_MULTITOUCH_TIMEOUT = 250; + +// Maximum time between press and release for a tap (ms) +const GH_TAP_TIMEOUT = 1000; + +// Timeout when waiting for longpress (ms) +const GH_LONGPRESS_TIMEOUT = 1000; + +// Timeout when waiting to decide between PINCH and TWODRAG (ms) +const GH_TWOTOUCH_TIMEOUT = 50; + +export default class GestureHandler { + constructor() { + this._target = null; + + this._state = GH_INITSTATE; + + this._tracked = []; + this._ignored = []; + + this._waitingRelease = false; + this._releaseStart = 0.0; + + this._longpressTimeoutId = null; + this._twoTouchTimeoutId = null; + + this._boundEventHandler = this._eventHandler.bind(this); + } + + attach(target) { + this.detach(); + + this._target = target; + this._target.addEventListener('touchstart', + this._boundEventHandler); + this._target.addEventListener('touchmove', + this._boundEventHandler); + this._target.addEventListener('touchend', + this._boundEventHandler); + this._target.addEventListener('touchcancel', + this._boundEventHandler); + } + + detach() { + if (!this._target) { + return; + } + + this._stopLongpressTimeout(); + this._stopTwoTouchTimeout(); + + this._target.removeEventListener('touchstart', + this._boundEventHandler); + this._target.removeEventListener('touchmove', + this._boundEventHandler); + this._target.removeEventListener('touchend', + this._boundEventHandler); + this._target.removeEventListener('touchcancel', + this._boundEventHandler); + this._target = null; + } + + _eventHandler(e) { + let fn; + + e.stopPropagation(); + e.preventDefault(); + + switch (e.type) { + case 'touchstart': + fn = this._touchStart; + break; + case 'touchmove': + fn = this._touchMove; + break; + case 'touchend': + case 'touchcancel': + fn = this._touchEnd; + break; + } + + for (let i = 0; i < e.changedTouches.length; i++) { + let touch = e.changedTouches[i]; + fn.call(this, touch.identifier, touch.clientX, touch.clientY); + } + } + + _touchStart(id, x, y) { + // Ignore any new touches if there is already an active gesture, + // or we're in a cleanup state + if (this._hasDetectedGesture() || (this._state === GH_NOGESTURE)) { + this._ignored.push(id); + return; + } + + // Did it take too long between touches that we should no longer + // consider this a single gesture? + if ((this._tracked.length > 0) && + ((Date.now() - this._tracked[0].started) > GH_MULTITOUCH_TIMEOUT)) { + this._state = GH_NOGESTURE; + this._ignored.push(id); + return; + } + + // If we're waiting for fingers to release then we should no longer + // recognize new touches + if (this._waitingRelease) { + this._state = GH_NOGESTURE; + this._ignored.push(id); + return; + } + + this._tracked.push({ + id: id, + started: Date.now(), + active: true, + firstX: x, + firstY: y, + lastX: x, + lastY: y, + angle: 0 + }); + + switch (this._tracked.length) { + case 1: + this._startLongpressTimeout(); + break; + + case 2: + this._state &= ~(GH_ONETAP | GH_DRAG | GH_LONGPRESS); + this._stopLongpressTimeout(); + break; + + case 3: + this._state &= ~(GH_TWOTAP | GH_TWODRAG | GH_PINCH); + break; + + default: + this._state = GH_NOGESTURE; + } + } + + _touchMove(id, x, y) { + let touch = this._tracked.find(t => t.id === id); + + // If this is an update for a touch we're not tracking, ignore it + if (touch === undefined) { + return; + } + + // Update the touches last position with the event coordinates + touch.lastX = x; + touch.lastY = y; + + let deltaX = x - touch.firstX; + let deltaY = y - touch.firstY; + + // Update angle when the touch has moved + if ((touch.firstX !== touch.lastX) || + (touch.firstY !== touch.lastY)) { + touch.angle = Math.atan2(deltaY, deltaX) * 180 / Math.PI; + } + + if (!this._hasDetectedGesture()) { + // Ignore moves smaller than the minimum threshold + if (Math.hypot(deltaX, deltaY) < GH_MOVE_THRESHOLD) { + return; + } + + // Can't be a tap or long press as we've seen movement + this._state &= ~(GH_ONETAP | GH_TWOTAP | GH_THREETAP | GH_LONGPRESS); + this._stopLongpressTimeout(); + + if (this._tracked.length !== 1) { + this._state &= ~(GH_DRAG); + } + if (this._tracked.length !== 2) { + this._state &= ~(GH_TWODRAG | GH_PINCH); + } + + // We need to figure out which of our different two touch gestures + // this might be + if (this._tracked.length === 2) { + + // The other touch is the one where the id doesn't match + let prevTouch = this._tracked.find(t => t.id !== id); + + // How far the previous touch point has moved since start + let prevDeltaMove = Math.hypot(prevTouch.firstX - prevTouch.lastX, + prevTouch.firstY - prevTouch.lastY); + + // We know that the current touch moved far enough, + // but unless both touches moved further than their + // threshold we don't want to disqualify any gestures + if (prevDeltaMove > GH_MOVE_THRESHOLD) { + + // The angle difference between the direction of the touch points + let deltaAngle = Math.abs(touch.angle - prevTouch.angle); + deltaAngle = Math.abs(((deltaAngle + 180) % 360) - 180); + + // PINCH or TWODRAG can be eliminated depending on the angle + if (deltaAngle > GH_ANGLE_THRESHOLD) { + this._state &= ~GH_TWODRAG; + } else { + this._state &= ~GH_PINCH; + } + + if (this._isTwoTouchTimeoutRunning()) { + this._stopTwoTouchTimeout(); + } + } else if (!this._isTwoTouchTimeoutRunning()) { + // We can't determine the gesture right now, let's + // wait and see if more events are on their way + this._startTwoTouchTimeout(); + } + } + + if (!this._hasDetectedGesture()) { + return; + } + + this._pushEvent('gesturestart'); + } + + this._pushEvent('gesturemove'); + } + + _touchEnd(id, x, y) { + // Check if this is an ignored touch + if (this._ignored.indexOf(id) !== -1) { + // Remove this touch from ignored + this._ignored.splice(this._ignored.indexOf(id), 1); + + // And reset the state if there are no more touches + if ((this._ignored.length === 0) && + (this._tracked.length === 0)) { + this._state = GH_INITSTATE; + this._waitingRelease = false; + } + return; + } + + // We got a touchend before the timer triggered, + // this cannot result in a gesture anymore. + if (!this._hasDetectedGesture() && + this._isTwoTouchTimeoutRunning()) { + this._stopTwoTouchTimeout(); + this._state = GH_NOGESTURE; + } + + // Some gestures don't trigger until a touch is released + if (!this._hasDetectedGesture()) { + // Can't be a gesture that relies on movement + this._state &= ~(GH_DRAG | GH_TWODRAG | GH_PINCH); + // Or something that relies on more time + this._state &= ~GH_LONGPRESS; + this._stopLongpressTimeout(); + + if (!this._waitingRelease) { + this._releaseStart = Date.now(); + this._waitingRelease = true; + + // Can't be a tap that requires more touches than we current have + switch (this._tracked.length) { + case 1: + this._state &= ~(GH_TWOTAP | GH_THREETAP); + break; + + case 2: + this._state &= ~(GH_ONETAP | GH_THREETAP); + break; + } + } + } + + // Waiting for all touches to release? (i.e. some tap) + if (this._waitingRelease) { + // Were all touches released at roughly the same time? + if ((Date.now() - this._releaseStart) > GH_MULTITOUCH_TIMEOUT) { + this._state = GH_NOGESTURE; + } + + // Did too long time pass between press and release? + if (this._tracked.some(t => (Date.now() - t.started) > GH_TAP_TIMEOUT)) { + this._state = GH_NOGESTURE; + } + + let touch = this._tracked.find(t => t.id === id); + touch.active = false; + + // Are we still waiting for more releases? + if (this._hasDetectedGesture()) { + this._pushEvent('gesturestart'); + } else { + // Have we reached a dead end? + if (this._state !== GH_NOGESTURE) { + return; + } + } + } + + if (this._hasDetectedGesture()) { + this._pushEvent('gestureend'); + } + + // Ignore any remaining touches until they are ended + for (let i = 0; i < this._tracked.length; i++) { + if (this._tracked[i].active) { + this._ignored.push(this._tracked[i].id); + } + } + this._tracked = []; + + this._state = GH_NOGESTURE; + + // Remove this touch from ignored if it's in there + if (this._ignored.indexOf(id) !== -1) { + this._ignored.splice(this._ignored.indexOf(id), 1); + } + + // We reset the state if ignored is empty + if ((this._ignored.length === 0)) { + this._state = GH_INITSTATE; + this._waitingRelease = false; + } + } + + _hasDetectedGesture() { + if (this._state === GH_NOGESTURE) { + return false; + } + // Check to see if the bitmask value is a power of 2 + // (i.e. only one bit set). If it is, we have a state. + if (this._state & (this._state - 1)) { + return false; + } + + // For taps we also need to have all touches released + // before we've fully detected the gesture + if (this._state & (GH_ONETAP | GH_TWOTAP | GH_THREETAP)) { + if (this._tracked.some(t => t.active)) { + return false; + } + } + + return true; + } + + _startLongpressTimeout() { + this._stopLongpressTimeout(); + this._longpressTimeoutId = setTimeout(() => this._longpressTimeout(), + GH_LONGPRESS_TIMEOUT); + } + + _stopLongpressTimeout() { + clearTimeout(this._longpressTimeoutId); + this._longpressTimeoutId = null; + } + + _longpressTimeout() { + if (this._hasDetectedGesture()) { + throw new Error("A longpress gesture failed, conflict with a different gesture"); + } + + this._state = GH_LONGPRESS; + this._pushEvent('gesturestart'); + } + + _startTwoTouchTimeout() { + this._stopTwoTouchTimeout(); + this._twoTouchTimeoutId = setTimeout(() => this._twoTouchTimeout(), + GH_TWOTOUCH_TIMEOUT); + } + + _stopTwoTouchTimeout() { + clearTimeout(this._twoTouchTimeoutId); + this._twoTouchTimeoutId = null; + } + + _isTwoTouchTimeoutRunning() { + return this._twoTouchTimeoutId !== null; + } + + _twoTouchTimeout() { + if (this._tracked.length === 0) { + throw new Error("A pinch or two drag gesture failed, no tracked touches"); + } + + // How far each touch point has moved since start + let avgM = this._getAverageMovement(); + let avgMoveH = Math.abs(avgM.x); + let avgMoveV = Math.abs(avgM.y); + + // The difference in the distance between where + // the touch points started and where they are now + let avgD = this._getAverageDistance(); + let deltaTouchDistance = Math.abs(Math.hypot(avgD.first.x, avgD.first.y) - + Math.hypot(avgD.last.x, avgD.last.y)); + + if ((avgMoveV < deltaTouchDistance) && + (avgMoveH < deltaTouchDistance)) { + this._state = GH_PINCH; + } else { + this._state = GH_TWODRAG; + } + + this._pushEvent('gesturestart'); + this._pushEvent('gesturemove'); + } + + _pushEvent(type) { + let detail = { type: this._stateToGesture(this._state) }; + + // For most gesture events the current (average) position is the + // most useful + let avg = this._getPosition(); + let pos = avg.last; + + // However we have a slight distance to detect gestures, so for the + // first gesture event we want to use the first positions we saw + if (type === 'gesturestart') { + pos = avg.first; + } + + // For these gestures, we always want the event coordinates + // to be where the gesture began, not the current touch location. + switch (this._state) { + case GH_TWODRAG: + case GH_PINCH: + pos = avg.first; + break; + } + + detail['clientX'] = pos.x; + detail['clientY'] = pos.y; + + // FIXME: other coordinates? + + // Some gestures also have a magnitude + if (this._state === GH_PINCH) { + let distance = this._getAverageDistance(); + if (type === 'gesturestart') { + detail['magnitudeX'] = distance.first.x; + detail['magnitudeY'] = distance.first.y; + } else { + detail['magnitudeX'] = distance.last.x; + detail['magnitudeY'] = distance.last.y; + } + } else if (this._state === GH_TWODRAG) { + if (type === 'gesturestart') { + detail['magnitudeX'] = 0.0; + detail['magnitudeY'] = 0.0; + } else { + let movement = this._getAverageMovement(); + detail['magnitudeX'] = movement.x; + detail['magnitudeY'] = movement.y; + } + } + + let gev = new CustomEvent(type, { detail: detail }); + this._target.dispatchEvent(gev); + } + + _stateToGesture(state) { + switch (state) { + case GH_ONETAP: + return 'onetap'; + case GH_TWOTAP: + return 'twotap'; + case GH_THREETAP: + return 'threetap'; + case GH_DRAG: + return 'drag'; + case GH_LONGPRESS: + return 'longpress'; + case GH_TWODRAG: + return 'twodrag'; + case GH_PINCH: + return 'pinch'; + } + + throw new Error("Unknown gesture state: " + state); + } + + _getPosition() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture position, no tracked touches"); + } + + let size = this._tracked.length; + let fx = 0, fy = 0, lx = 0, ly = 0; + + for (let i = 0; i < this._tracked.length; i++) { + fx += this._tracked[i].firstX; + fy += this._tracked[i].firstY; + lx += this._tracked[i].lastX; + ly += this._tracked[i].lastY; + } + + return { first: { x: fx / size, + y: fy / size }, + last: { x: lx / size, + y: ly / size } }; + } + + _getAverageMovement() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture movement, no tracked touches"); + } + + let totalH, totalV; + totalH = totalV = 0; + let size = this._tracked.length; + + for (let i = 0; i < this._tracked.length; i++) { + totalH += this._tracked[i].lastX - this._tracked[i].firstX; + totalV += this._tracked[i].lastY - this._tracked[i].firstY; + } + + return { x: totalH / size, + y: totalV / size }; + } + + _getAverageDistance() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture distance, no tracked touches"); + } + + // Distance between the first and last tracked touches + + let first = this._tracked[0]; + let last = this._tracked[this._tracked.length - 1]; + + let fdx = Math.abs(last.firstX - first.firstX); + let fdy = Math.abs(last.firstY - first.firstY); + + let ldx = Math.abs(last.lastX - first.lastX); + let ldy = Math.abs(last.lastY - first.lastY); + + return { first: { x: fdx, y: fdy }, + last: { x: ldx, y: ldy } }; + } +} diff --git a/core/input/mouse.js b/core/input/mouse.js index 5f1b6b4b..82ea335a 100644 --- a/core/input/mouse.js +++ b/core/input/mouse.js @@ -5,7 +5,6 @@ */ import * as Log from '../util/logging.js'; -import { isTouchDevice } from '../util/browser.js'; import { setCapture, stopEvent, getPointerEvent } from '../util/events.js'; const WHEEL_STEP = 10; // Delta threshold for a mouse wheel step @@ -16,9 +15,6 @@ export default class Mouse { constructor(target) { this._target = target || document; - this._doubleClickTimer = null; - this._lastTouchPos = null; - this._pos = null; this._wheelStepXTimer = null; this._wheelStepYTimer = null; @@ -33,11 +29,6 @@ export default class Mouse { 'mousedisable': this._handleMouseDisable.bind(this) }; - // ===== PROPERTIES ===== - - this.touchButton = 1; // Button mask (1, 2, 4) for touch devices - // (0 means ignore clicks) - // ===== EVENT HANDLERS ===== this.onmousebutton = () => {}; // Handler for mouse button press/release @@ -55,39 +46,7 @@ export default class Mouse { let pos = this._pos; let bmask; - if (e.touches || e.changedTouches) { - // Touch device - - // When two touches occur within 500 ms of each other and are - // close enough together a double click is triggered. - if (down == 1) { - if (this._doubleClickTimer === null) { - this._lastTouchPos = pos; - } else { - clearTimeout(this._doubleClickTimer); - - // When the distance between the two touches is small enough - // force the position of the latter touch to the position of - // the first. - - const xs = this._lastTouchPos.x - pos.x; - const ys = this._lastTouchPos.y - pos.y; - const d = Math.sqrt((xs * xs) + (ys * ys)); - - // The goal is to trigger on a certain physical width, - // the devicePixelRatio brings us a bit closer but is - // not optimal. - const threshold = 20 * (window.devicePixelRatio || 1); - if (d < threshold) { - pos = this._lastTouchPos; - } - } - this._doubleClickTimer = - setTimeout(this._resetDoubleClickTimer.bind(this), 500); - } - bmask = this.touchButton; - // If bmask is set - } else if (e.which) { + if (e.which) { /* everything except IE */ bmask = 1 << e.button; } else { @@ -105,10 +64,7 @@ export default class Mouse { } _handleMouseDown(e) { - // Touch events have implicit capture - if (e.type === "mousedown") { - setCapture(this._target); - } + setCapture(this._target); this._handleMouseButton(e, 1); } @@ -242,11 +198,6 @@ export default class Mouse { grab() { const t = this._target; - if (isTouchDevice) { - t.addEventListener('touchstart', this._eventHandlers.mousedown); - t.addEventListener('touchend', this._eventHandlers.mouseup); - t.addEventListener('touchmove', this._eventHandlers.mousemove); - } t.addEventListener('mousedown', this._eventHandlers.mousedown); t.addEventListener('mouseup', this._eventHandlers.mouseup); t.addEventListener('mousemove', this._eventHandlers.mousemove); @@ -265,11 +216,6 @@ export default class Mouse { this._resetWheelStepTimers(); - if (isTouchDevice) { - t.removeEventListener('touchstart', this._eventHandlers.mousedown); - t.removeEventListener('touchend', this._eventHandlers.mouseup); - t.removeEventListener('touchmove', this._eventHandlers.mousemove); - } t.removeEventListener('mousedown', this._eventHandlers.mousedown); t.removeEventListener('mouseup', this._eventHandlers.mouseup); t.removeEventListener('mousemove', this._eventHandlers.mousemove); diff --git a/core/rfb.js b/core/rfb.js index ca50779e..9db375fc 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -11,12 +11,14 @@ import { toUnsigned32bit, toSigned32bit } from './util/int.js'; import * as Log from './util/logging.js'; import { encodeUTF8, decodeUTF8 } from './util/strings.js'; import { dragThreshold } from './util/browser.js'; +import { clientToElement } from './util/element.js'; import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Inflator from "./inflator.js"; import Deflator from "./deflator.js"; import Keyboard from "./input/keyboard.js"; import Mouse from "./input/mouse.js"; +import GestureHandler from "./input/gesturehandler.js"; import Cursor from "./util/cursor.js"; import Websock from "./websock.js"; import DES from "./des.js"; @@ -39,6 +41,12 @@ const DEFAULT_BACKGROUND = 'rgb(40, 40, 40)'; // Minimum wait (ms) between two mouse moves const MOUSE_MOVE_DELAY = 17; +// Gesture thresholds +const GESTURE_ZOOMSENS = 75; +const GESTURE_SCRLSENS = 50; +const DOUBLE_TAP_TIMEOUT = 1000; +const DOUBLE_TAP_THRESHOLD = 50; + // Extended clipboard pseudo-encoding formats const extendedClipboardFormatText = 1; /*eslint-disable no-unused-vars */ @@ -118,6 +126,7 @@ export default class RFB extends EventTargetMixin { this._flushing = false; // Display flushing state this._keyboard = null; // Keyboard input handler object this._mouse = null; // Mouse input handler object + this._gestures = null; // Gesture input handler object // Timers this._disconnTimer = null; // disconnection timer @@ -144,10 +153,17 @@ export default class RFB extends EventTargetMixin { this._viewportDragPos = {}; this._viewportHasMoved = false; + // Gesture state + this._gestureLastTapTime = null; + this._gestureFirstDoubleTapEv = null; + this._gestureLastMagnitudeX = 0; + this._gestureLastMagnitudeY = 0; + // Bound event handlers this._eventHandlers = { focusCanvas: this._focusCanvas.bind(this), windowResize: this._windowResize.bind(this), + handleGesture: this._handleGesture.bind(this), }; // main setup @@ -210,6 +226,8 @@ export default class RFB extends EventTargetMixin { this._mouse.onmousebutton = this._handleMouseButton.bind(this); this._mouse.onmousemove = this._handleMouseMove.bind(this); + this._gestures = new GestureHandler(); + this._sock = new Websock(); this._sock.on('message', () => { this._handleMessage(); @@ -306,8 +324,8 @@ export default class RFB extends EventTargetMixin { get capabilities() { return this._capabilities; } - get touchButton() { return this._mouse.touchButton; } - set touchButton(button) { this._mouse.touchButton = button; } + get touchButton() { return 0; } + set touchButton(button) { Log.Warn("Using old API!"); } get clipViewport() { return this._clipViewport; } set clipViewport(viewport) { @@ -501,6 +519,8 @@ export default class RFB extends EventTargetMixin { // Make our elements part of the page this._target.appendChild(this._screen); + this._gestures.attach(this._canvas); + this._cursor.attach(this._canvas); this._refreshCursor(); @@ -512,17 +532,26 @@ export default class RFB extends EventTargetMixin { this._canvas.addEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.addEventListener("touchstart", this._eventHandlers.focusCanvas); + // Gesture events + this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture); + this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture); + this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture); + Log.Debug("<< RFB.connect"); } _disconnect() { Log.Debug(">> RFB.disconnect"); this._cursor.detach(); + this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture); + this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture); + this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); this._keyboard.ungrab(); this._mouse.ungrab(); + this._gestures.detach(); this._sock.close(); try { this._target.removeChild(this._screen); @@ -910,6 +939,156 @@ export default class RFB extends EventTargetMixin { this._display.absY(y), mask); } + _handleTapEvent(ev, bmask) { + let pos = clientToElement(ev.detail.clientX, ev.detail.clientY, + this._canvas); + + // If the user quickly taps multiple times we assume they meant to + // hit the same spot, so slightly adjust coordinates + + if ((this._gestureLastTapTime !== null) && + ((Date.now() - this._gestureLastTapTime) < DOUBLE_TAP_TIMEOUT) && + (this._gestureFirstDoubleTapEv.detail.type === ev.detail.type)) { + let dx = this._gestureFirstDoubleTapEv.detail.clientX - ev.detail.clientX; + let dy = this._gestureFirstDoubleTapEv.detail.clientY - ev.detail.clientY; + let distance = Math.hypot(dx, dy); + + if (distance < DOUBLE_TAP_THRESHOLD) { + pos = clientToElement(this._gestureFirstDoubleTapEv.detail.clientX, + this._gestureFirstDoubleTapEv.detail.clientY, + this._canvas); + } else { + this._gestureFirstDoubleTapEv = ev; + } + } else { + this._gestureFirstDoubleTapEv = ev; + } + this._gestureLastTapTime = Date.now(); + + this._handleMouseMove(pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, true, bmask); + this._handleMouseButton(pos.x, pos.y, false, bmask); + } + + _handleGesture(ev) { + let magnitude; + + let pos = clientToElement(ev.detail.clientX, ev.detail.clientY, + this._canvas); + switch (ev.type) { + case 'gesturestart': + switch (ev.detail.type) { + case 'onetap': + this._handleTapEvent(ev, 0x1); + break; + case 'twotap': + this._handleTapEvent(ev, 0x4); + break; + case 'threetap': + this._handleTapEvent(ev, 0x2); + break; + case 'drag': + this._handleMouseMove(pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, true, 0x1); + break; + case 'longpress': + this._handleMouseMove(pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, true, 0x4); + break; + + case 'twodrag': + this._gestureLastMagnitudeX = ev.detail.magnitudeX; + this._gestureLastMagnitudeY = ev.detail.magnitudeY; + this._handleMouseMove(pos.x, pos.y); + break; + case 'pinch': + this._gestureLastMagnitudeX = Math.hypot(ev.detail.magnitudeX, + ev.detail.magnitudeY); + this._handleMouseMove(pos.x, pos.y); + break; + } + break; + + case 'gesturemove': + switch (ev.detail.type) { + case 'onetap': + case 'twotap': + case 'threetap': + break; + case 'drag': + case 'longpress': + this._handleMouseMove(pos.x, pos.y); + break; + case 'twodrag': + // Always scroll in the same position. + // We don't know if the mouse was moved so we need to move it + // every update. + this._handleMouseMove(pos.x, pos.y); + while ((ev.detail.magnitudeY - this._gestureLastMagnitudeY) > GESTURE_SCRLSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x8); + this._handleMouseButton(pos.x, pos.y, false, 0x8); + this._gestureLastMagnitudeY += GESTURE_SCRLSENS; + } + while ((ev.detail.magnitudeY - this._gestureLastMagnitudeY) < -GESTURE_SCRLSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x10); + this._handleMouseButton(pos.x, pos.y, false, 0x10); + this._gestureLastMagnitudeY -= GESTURE_SCRLSENS; + } + while ((ev.detail.magnitudeX - this._gestureLastMagnitudeX) > GESTURE_SCRLSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x20); + this._handleMouseButton(pos.x, pos.y, false, 0x20); + this._gestureLastMagnitudeX += GESTURE_SCRLSENS; + } + while ((ev.detail.magnitudeX - this._gestureLastMagnitudeX) < -GESTURE_SCRLSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x40); + this._handleMouseButton(pos.x, pos.y, false, 0x40); + this._gestureLastMagnitudeX -= GESTURE_SCRLSENS; + } + break; + case 'pinch': + // Always scroll in the same position. + // We don't know if the mouse was moved so we need to move it + // every update. + this._handleMouseMove(pos.x, pos.y); + magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY); + if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { + this._handleKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); + while ((magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x8); + this._handleMouseButton(pos.x, pos.y, false, 0x8); + this._gestureLastMagnitudeX += GESTURE_ZOOMSENS; + } + while ((magnitude - this._gestureLastMagnitudeX) < -GESTURE_ZOOMSENS) { + this._handleMouseButton(pos.x, pos.y, true, 0x10); + this._handleMouseButton(pos.x, pos.y, false, 0x10); + this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS; + } + } + this._handleKeyEvent(KeyTable.XK_Control_L, "ControlLeft", false); + } + break; + + case 'gestureend': + switch (ev.detail.type) { + case 'onetap': + case 'twotap': + case 'threetap': + case 'pinch': + case 'twodrag': + break; + case 'drag': + this._handleMouseMove(pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, false, 0x1); + break; + case 'longpress': + this._handleMouseMove(pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, false, 0x4); + break; + } + break; + } + } + // Message Handlers _negotiateProtocolVersion() { diff --git a/core/util/element.js b/core/util/element.js new file mode 100644 index 00000000..466a7453 --- /dev/null +++ b/core/util/element.js @@ -0,0 +1,32 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2020 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + */ + +/* + * HTML element utility functions + */ + +export function clientToElement(x, y, elem) { + const bounds = elem.getBoundingClientRect(); + let pos = { x: 0, y: 0 }; + // Clip to target bounds + if (x < bounds.left) { + pos.x = 0; + } else if (x >= bounds.right) { + pos.x = bounds.width - 1; + } else { + pos.x = x - bounds.left; + } + if (y < bounds.top) { + pos.y = 0; + } else if (y >= bounds.bottom) { + pos.y = bounds.height - 1; + } else { + pos.y = y - bounds.top; + } + return pos; +} diff --git a/docs/API.md b/docs/API.md index 59b7cf17..349fdc88 100644 --- a/docs/API.md +++ b/docs/API.md @@ -26,12 +26,6 @@ protocol stream. moved to the remote session when a `mousedown` or `touchstart` event is received. Enabled by default. -`touchButton` - - Is a `long` controlling the button mask that should be simulated - when a touch event is recieved. Uses the same values as - [`MouseEvent.button`](https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/button). - Is set to `1` by default. - `clipViewport` - Is a `boolean` indicating if the remote session should be clipped to its container. When disabled scrollbars will be shown to handle diff --git a/tests/test.gesturehandler.js b/tests/test.gesturehandler.js new file mode 100644 index 00000000..bfb26219 --- /dev/null +++ b/tests/test.gesturehandler.js @@ -0,0 +1,1031 @@ +const expect = chai.expect; + +import EventTargetMixin from '../core/util/eventtarget.js'; + +import GestureHandler from '../core/input/gesturehandler.js'; +import * as browser from '../core/util/browser.js'; + +class DummyTarget extends EventTargetMixin { +} + +describe('Gesture handler', function () { + let target, handler; + let gestures; + let clock; + let touches; + + before(function () { + clock = sinon.useFakeTimers(); + }); + + after(function () { + clock.restore(); + }); + + beforeEach(function () { + // Touch events and gestures are not supported on IE + if (browser.isIE()) { + this.skip(); + return; + } + + target = new DummyTarget(); + gestures = sinon.spy(); + target.addEventListener('gesturestart', gestures); + target.addEventListener('gesturemove', gestures); + target.addEventListener('gestureend', gestures); + touches = []; + handler = new GestureHandler(); + handler.attach(target); + }); + + afterEach(function () { + handler.detach(); + target = null; + gestures = null; + }); + + function touchStart(id, x, y) { + let touch = { identifier: id, + clientX: x, clientY: y }; + touches.push(touch); + let ev = { type: 'touchstart', + touches: touches, + targetTouches: touches, + changedTouches: [ touch ], + stopPropagation: sinon.spy(), + preventDefault: sinon.spy() }; + target.dispatchEvent(ev); + } + + function touchMove(id, x, y) { + let touch = touches.find(t => t.identifier === id); + touch.clientX = x; + touch.clientY = y; + let ev = { type: 'touchmove', + touches: touches, + targetTouches: touches, + changedTouches: [ touch ], + stopPropagation: sinon.spy(), + preventDefault: sinon.spy() }; + target.dispatchEvent(ev); + } + + function touchEnd(id) { + let idx = touches.findIndex(t => t.identifier === id); + let touch = touches.splice(idx, 1)[0]; + let ev = { type: 'touchend', + touches: touches, + targetTouches: touches, + changedTouches: [ touch ], + stopPropagation: sinon.spy(), + preventDefault: sinon.spy() }; + target.dispatchEvent(ev); + } + + describe('Single finger tap', function () { + it('should handle single finger tap', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchEnd(1); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'onetap', + clientX: 20.0, + clientY: 30.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'onetap', + clientX: 20.0, + clientY: 30.0 } })); + }); + }); + + describe('Two finger tap', function () { + it('should handle two finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + + expect(gestures).to.not.have.been.called; + + touchEnd(1); + + expect(gestures).to.not.have.been.called; + + touchEnd(2); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twotap', + clientX: 25.0, + clientY: 40.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'twotap', + clientX: 25.0, + clientY: 40.0 } })); + }); + + it('should ignore slow starting two finger tap', function () { + touchStart(1, 20.0, 30.0); + + clock.tick(500); + + touchStart(2, 30.0, 50.0); + touchEnd(1); + touchEnd(2); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore slow ending two finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + touchEnd(1); + + clock.tick(500); + + touchEnd(2); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore slow two finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + + clock.tick(1500); + + touchEnd(1); + touchEnd(2); + + expect(gestures).to.not.have.been.called; + }); + }); + + describe('Three finger tap', function () { + it('should handle three finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + touchStart(3, 40.0, 40.0); + + expect(gestures).to.not.have.been.called; + + touchEnd(1); + + expect(gestures).to.not.have.been.called; + + touchEnd(2); + + expect(gestures).to.not.have.been.called; + + touchEnd(3); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'threetap', + clientX: 30.0, + clientY: 40.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'threetap', + clientX: 30.0, + clientY: 40.0 } })); + }); + + it('should ignore slow starting three finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + + clock.tick(500); + + touchStart(3, 40.0, 40.0); + touchEnd(1); + touchEnd(2); + touchEnd(3); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore slow ending three finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + touchStart(3, 40.0, 40.0); + touchEnd(1); + touchEnd(2); + + clock.tick(500); + + touchEnd(3); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore three finger drag', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + touchStart(3, 40.0, 40.0); + + touchMove(1, 120.0, 130.0); + touchMove(2, 130.0, 150.0); + touchMove(3, 140.0, 140.0); + + touchEnd(1); + touchEnd(2); + touchEnd(3); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore slow three finger tap', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 50.0); + touchStart(3, 40.0, 40.0); + + clock.tick(1500); + + touchEnd(1); + touchEnd(2); + touchEnd(3); + + expect(gestures).to.not.have.been.called; + }); + }); + + describe('Single finger drag', function () { + it('should handle horizontal single finger drag', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 40.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 80.0, 30.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'drag', + clientX: 20.0, + clientY: 30.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag', + clientX: 80.0, + clientY: 30.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'drag', + clientX: 80.0, + clientY: 30.0 } })); + }); + + it('should handle vertical single finger drag', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 20.0, 50.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 20.0, 90.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'drag', + clientX: 20.0, + clientY: 30.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag', + clientX: 20.0, + clientY: 90.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'drag', + clientX: 20.0, + clientY: 90.0 } })); + }); + + it('should handle diagonal single finger drag', function () { + touchStart(1, 120.0, 130.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 90.0, 100.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 60.0, 70.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'drag', + clientX: 120.0, + clientY: 130.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag', + clientX: 60.0, + clientY: 70.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'drag', + clientX: 60.0, + clientY: 70.0 } })); + }); + }); + + describe('Long press', function () { + it('should handle long press', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(1500); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'longpress', + clientX: 20.0, + clientY: 30.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'longpress', + clientX: 20.0, + clientY: 30.0 } })); + }); + + it('should handle long press drag', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(1500); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'longpress', + clientX: 20.0, + clientY: 30.0 } })); + + gestures.resetHistory(); + + touchMove(1, 120.0, 50.0); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'longpress', + clientX: 120.0, + clientY: 50.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'longpress', + clientX: 120.0, + clientY: 50.0 } })); + }); + }); + + describe('Two finger drag', function () { + it('should handle fast and distinct horizontal two finger drag', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 40.0, 30.0); + touchMove(2, 50.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(2, 90.0, 30.0); + touchMove(1, 80.0, 30.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 60.0, + magnitudeY: 0.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 60.0, + magnitudeY: 0.0 } })); + }); + + it('should handle fast and distinct vertical two finger drag', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 20.0, 100.0); + touchMove(2, 30.0, 40.0); + + expect(gestures).to.not.have.been.called; + + touchMove(2, 30.0, 90.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 0.0, + magnitudeY: 65.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'twodrag', + clientX: 25.0, + clientY: 30.0, + magnitudeX: 0.0, + magnitudeY: 65.0 } })); + }); + + it('should handle fast and distinct diagonal two finger drag', function () { + touchStart(1, 120.0, 130.0); + touchStart(2, 130.0, 130.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 80.0, 90.0); + touchMove(2, 100.0, 130.0); + + expect(gestures).to.not.have.been.called; + + touchMove(2, 60.0, 70.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 125.0, + clientY: 130.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 125.0, + clientY: 130.0, + magnitudeX: -55.0, + magnitudeY: -50.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'twodrag', + clientX: 125.0, + clientY: 130.0, + magnitudeX: -55.0, + magnitudeY: -50.0 } })); + }); + + it('should ignore fast almost two finger dragging', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 30.0); + touchMove(1, 80.0, 30.0); + touchMove(2, 70.0, 30.0); + touchEnd(1); + touchEnd(2); + + expect(gestures).to.not.have.been.called; + + clock.tick(1500); + + expect(gestures).to.not.have.been.called; + }); + + it('should handle slow horizontal two finger drag', function () { + touchStart(1, 50.0, 40.0); + touchStart(2, 60.0, 40.0); + touchMove(1, 80.0, 40.0); + touchMove(2, 110.0, 40.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(60); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 55.0, + clientY: 40.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 55.0, + clientY: 40.0, + magnitudeX: 40.0, + magnitudeY: 0.0 } })); + }); + + it('should handle slow vertical two finger drag', function () { + touchStart(1, 40.0, 40.0); + touchStart(2, 40.0, 60.0); + touchMove(2, 40.0, 80.0); + touchMove(1, 40.0, 100.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(60); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 40.0, + clientY: 50.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 40.0, + clientY: 50.0, + magnitudeX: 0.0, + magnitudeY: 40.0 } })); + }); + + it('should handle slow diagonal two finger drag', function () { + touchStart(1, 50.0, 40.0); + touchStart(2, 40.0, 60.0); + touchMove(1, 70.0, 60.0); + touchMove(2, 90.0, 110.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(60); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag', + clientX: 45.0, + clientY: 50.0, + magnitudeX: 0.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag', + clientX: 45.0, + clientY: 50.0, + magnitudeX: 35.0, + magnitudeY: 35.0 } })); + }); + + it('should ignore too slow two finger drag', function () { + touchStart(1, 20.0, 30.0); + + clock.tick(500); + + touchStart(2, 30.0, 30.0); + touchMove(1, 40.0, 30.0); + touchMove(2, 50.0, 30.0); + touchMove(1, 80.0, 30.0); + + expect(gestures).to.not.have.been.called; + }); + }); + + describe('Pinch', function () { + it('should handle pinching distinctly and fast inwards', function () { + touchStart(1, 0.0, 0.0); + touchStart(2, 130.0, 130.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 50.0, 40.0); + touchMove(2, 100.0, 130.0); + + expect(gestures).to.not.have.been.called; + + touchMove(2, 60.0, 70.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'pinch', + clientX: 65.0, + clientY: 65.0, + magnitudeX: 130.0, + magnitudeY: 130.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'pinch', + clientX: 65.0, + clientY: 65.0, + magnitudeX: 10.0, + magnitudeY: 30.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'pinch', + clientX: 65.0, + clientY: 65.0, + magnitudeX: 10.0, + magnitudeY: 30.0 } })); + }); + + it('should handle pinching fast and distinctly outwards', function () { + touchStart(1, 100.0, 100.0); + touchStart(2, 110.0, 100.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 130.0, 70.0); + touchMove(2, 0.0, 200.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 180.0, 20.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'pinch', + clientX: 105.0, + clientY: 100.0, + magnitudeX: 10.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'pinch', + clientX: 105.0, + clientY: 100.0, + magnitudeX: 180.0, + magnitudeY: 180.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'pinch', + clientX: 105.0, + clientY: 100.0, + magnitudeX: 180.0, + magnitudeY: 180.0 } })); + }); + + it('should ignore fast almost pinching', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 130.0, 130.0); + touchMove(1, 80.0, 70.0); + touchEnd(1); + touchEnd(2); + + expect(gestures).to.not.have.been.called; + + clock.tick(1500); + + expect(gestures).to.not.have.been.called; + }); + + it('should handle pinching inwards slowly', function () { + touchStart(1, 0.0, 0.0); + touchStart(2, 130.0, 130.0); + touchMove(1, 50.0, 40.0); + touchMove(2, 100.0, 130.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(60); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'pinch', + clientX: 65.0, + clientY: 65.0, + magnitudeX: 130.0, + magnitudeY: 130.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'pinch', + clientX: 65.0, + clientY: 65.0, + magnitudeX: 50.0, + magnitudeY: 90.0 } })); + }); + + it('should handle pinching outwards slowly', function () { + touchStart(1, 100.0, 130.0); + touchStart(2, 110.0, 130.0); + touchMove(2, 200.0, 130.0); + + expect(gestures).to.not.have.been.called; + + clock.tick(60); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'pinch', + clientX: 105.0, + clientY: 130.0, + magnitudeX: 10.0, + magnitudeY: 0.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'pinch', + clientX: 105.0, + clientY: 130.0, + magnitudeX: 100.0, + magnitudeY: 0.0 } })); + }); + + it('should ignore pinching too slowly', function () { + touchStart(1, 0.0, 0.0); + + clock.tick(500); + + touchStart(2, 130.0, 130.0); + touchMove(2, 100.0, 130.0); + touchMove(1, 50.0, 40.0); + + expect(gestures).to.not.have.been.called; + }); + }); + + describe('Ignoring', function () { + it('should ignore extra touches during gesture', function () { + touchStart(1, 20.0, 30.0); + touchMove(1, 40.0, 30.0); + touchMove(1, 80.0, 30.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'drag' } })); + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag' } })); + + gestures.resetHistory(); + + touchStart(2, 10.0, 10.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 100.0, 50.0); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag', + clientX: 100.0, + clientY: 50.0 } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'drag', + clientX: 100.0, + clientY: 50.0 } })); + }); + + it('should ignore extra touches when waiting for gesture to end', function () { + touchStart(1, 20.0, 30.0); + touchStart(2, 30.0, 30.0); + touchMove(1, 40.0, 30.0); + touchMove(2, 90.0, 30.0); + touchMove(1, 80.0, 30.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'twodrag' } })); + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'twodrag' } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'twodrag' } })); + + gestures.resetHistory(); + + touchStart(3, 10.0, 10.0); + touchEnd(3); + + expect(gestures).to.not.have.been.called; + }); + + it('should ignore extra touches after gesture', function () { + touchStart(1, 20.0, 30.0); + touchMove(1, 40.0, 30.0); + touchMove(1, 80.0, 30.0); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'drag' } })); + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag' } })); + + gestures.resetHistory(); + + touchStart(2, 10.0, 10.0); + + expect(gestures).to.not.have.been.called; + + touchMove(1, 100.0, 50.0); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gesturemove', + detail: { type: 'drag' } })); + + gestures.resetHistory(); + + touchEnd(1); + + expect(gestures).to.have.been.calledOnceWith( + sinon.match({ type: 'gestureend', + detail: { type: 'drag' } })); + + gestures.resetHistory(); + + touchEnd(2); + + expect(gestures).to.not.have.been.called; + + // Check that everything is reseted after trailing ignores are released + + touchStart(3, 20.0, 30.0); + touchEnd(3); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'onetap' } })); + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'onetap' } })); + }); + + it('should properly reset after a gesture', function () { + touchStart(1, 20.0, 30.0); + + expect(gestures).to.not.have.been.called; + + touchEnd(1); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'onetap', + clientX: 20.0, + clientY: 30.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'onetap', + clientX: 20.0, + clientY: 30.0 } })); + + gestures.resetHistory(); + + touchStart(2, 70.0, 80.0); + + expect(gestures).to.not.have.been.called; + + touchEnd(2); + + expect(gestures).to.have.been.calledTwice; + + expect(gestures.firstCall).to.have.been.calledWith( + sinon.match({ type: 'gesturestart', + detail: { type: 'onetap', + clientX: 70.0, + clientY: 80.0 } })); + + expect(gestures.secondCall).to.have.been.calledWith( + sinon.match({ type: 'gestureend', + detail: { type: 'onetap', + clientX: 70.0, + clientY: 80.0 } })); + }); + }); +}); diff --git a/tests/test.mouse.js b/tests/test.mouse.js index 8e066c15..8830c4be 100644 --- a/tests/test.mouse.js +++ b/tests/test.mouse.js @@ -34,7 +34,6 @@ describe('Mouse Event Handling', function () { e.preventDefault = sinon.spy(); return e; }; - const touchevent = mouseevent; describe('Decode Mouse Events', function () { it('should decode mousedown events', function (done) { @@ -89,131 +88,6 @@ describe('Mouse Event Handling', function () { }); }); - describe('Double-click for Touch', function () { - - beforeEach(function () { this.clock = sinon.useFakeTimers(); }); - afterEach(function () { this.clock.restore(); }); - - it('should use same pos for 2nd tap if close enough', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - calls++; - if (calls === 1) { - expect(down).to.be.equal(1); - expect(x).to.be.equal(68); - expect(y).to.be.equal(36); - } else if (calls === 3) { - expect(down).to.be.equal(1); - expect(x).to.be.equal(68); - expect(y).to.be.equal(36); - done(); - } - }; - // touch events are sent in an array of events - // with one item for each touch point - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 78, clientY: 46 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 79, clientY: 45 }]})); - this.clock.tick(200); - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 67, clientY: 35 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 66, clientY: 36 }]})); - }); - - it('should not modify 2nd tap pos if far apart', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - calls++; - if (calls === 1) { - expect(down).to.be.equal(1); - expect(x).to.be.equal(68); - expect(y).to.be.equal(36); - } else if (calls === 3) { - expect(down).to.be.equal(1); - expect(x).to.not.be.equal(68); - expect(y).to.not.be.equal(36); - done(); - } - }; - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 78, clientY: 46 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 79, clientY: 45 }]})); - this.clock.tick(200); - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 57, clientY: 35 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 56, clientY: 36 }]})); - }); - - it('should not modify 2nd tap pos if not soon enough', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - calls++; - if (calls === 1) { - expect(down).to.be.equal(1); - expect(x).to.be.equal(68); - expect(y).to.be.equal(36); - } else if (calls === 3) { - expect(down).to.be.equal(1); - expect(x).to.not.be.equal(68); - expect(y).to.not.be.equal(36); - done(); - } - }; - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 78, clientY: 46 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 79, clientY: 45 }]})); - this.clock.tick(500); - mouse._handleMouseDown(touchevent( - 'touchstart', { touches: [{ clientX: 67, clientY: 35 }]})); - this.clock.tick(10); - mouse._handleMouseUp(touchevent( - 'touchend', { touches: [{ clientX: 66, clientY: 36 }]})); - }); - - it('should not modify 2nd tap pos if not touch', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - calls++; - if (calls === 1) { - expect(down).to.be.equal(1); - expect(x).to.be.equal(68); - expect(y).to.be.equal(36); - } else if (calls === 3) { - expect(down).to.be.equal(1); - expect(x).to.not.be.equal(68); - expect(y).to.not.be.equal(36); - done(); - } - }; - mouse._handleMouseDown(mouseevent( - 'mousedown', { button: '0x01', clientX: 78, clientY: 46 })); - this.clock.tick(10); - mouse._handleMouseUp(mouseevent( - 'mouseup', { button: '0x01', clientX: 79, clientY: 45 })); - this.clock.tick(200); - mouse._handleMouseDown(mouseevent( - 'mousedown', { button: '0x01', clientX: 67, clientY: 35 })); - this.clock.tick(10); - mouse._handleMouseUp(mouseevent( - 'mouseup', { button: '0x01', clientX: 66, clientY: 36 })); - }); - - }); - describe('Accumulate mouse wheel events with small delta', function () { beforeEach(function () { this.clock = sinon.useFakeTimers(); }); diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 62efefdb..e46f40c5 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -7,6 +7,8 @@ import { deflateInit, deflate } from "../vendor/pako/lib/zlib/deflate.js"; import { encodings } from '../core/encodings.js'; import { toUnsigned32bit } from '../core/util/int.js'; import { encodeUTF8 } from '../core/util/strings.js'; +import KeyTable from '../core/input/keysym.js'; +import * as browser from '../core/util/browser.js'; import FakeWebSocket from './fake.websocket.js'; @@ -2892,7 +2894,753 @@ describe('Remote Frame Buffer Protocol Client', function () { }); }); - describe('WebSocket event handlers', function () { + describe('Gesture event handlers', function () { + let pointerEvent; + + beforeEach(function () { + // Touch events and gestures are not supported on IE + if (browser.isIE()) { + this.skip(); + return; + } + + pointerEvent = sinon.spy(RFB.messages, 'pointerEvent'); + + client._display.resize(100, 100); + }); + + afterEach(function () { + pointerEvent.restore(); + }); + + function elementToClient(x, y) { + let res = { x: 0, y: 0 }; + + let bounds = client._canvas.getBoundingClientRect(); + + /* + * If the canvas is on a fractional position we will calculate + * a fractional mouse position. But that gets truncated when we + * send the event, AND the same thing happens in RFB when it + * generates the PointerEvent message. To compensate for that + * fact we round the value upwards here. + */ + res.x = Math.ceil(bounds.left + x); + res.y = Math.ceil(bounds.top + y); + + return res; + } + + function gestureStart(gestureType, x, y, + magnitudeX = 0, magnitudeY = 0) { + let pos = elementToClient(x, y); + let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; + + detail.magnitudeX = magnitudeX; + detail.magnitudeY = magnitudeY; + + let ev = new CustomEvent('gesturestart', { detail: detail }); + client._canvas.dispatchEvent(ev); + } + + function gestureMove(gestureType, x, y, + magnitudeX = 0, magnitudeY = 0) { + let pos = elementToClient(x, y); + let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; + + detail.magnitudeX = magnitudeX; + detail.magnitudeY = magnitudeY; + + let ev = new CustomEvent('gesturemove', { detail: detail }); + client._canvas.dispatchEvent(ev); + } + + function gestureEnd(gestureType, x, y) { + let pos = elementToClient(x, y); + let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; + let ev = new CustomEvent('gestureend', { detail: detail }); + client._canvas.dispatchEvent(ev); + } + + describe('Gesture onetap', function () { + it('should handle onetap events', function () { + let bmask = 0x1; + + gestureStart('onetap', 20, 40); + gestureEnd('onetap', 20, 40); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should keep same position for multiple onetap events', function () { + let bmask = 0x1; + + gestureStart('onetap', 20, 40); + gestureEnd('onetap', 20, 40); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureStart('onetap', 20, 50); + gestureEnd('onetap', 20, 50); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureStart('onetap', 30, 50); + gestureEnd('onetap', 30, 50); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should not keep same position for onetap events when too far apart', function () { + let bmask = 0x1; + + gestureStart('onetap', 20, 40); + gestureEnd('onetap', 20, 40); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureStart('onetap', 80, 95); + gestureEnd('onetap', 80, 95); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 80, 95, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 80, 95, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 80, 95, 0x0); + }); + + it('should not keep same position for onetap events when enough time inbetween', function () { + let bmask = 0x1; + + gestureStart('onetap', 10, 20); + gestureEnd('onetap', 10, 20); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 20, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 20, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 10, 20, 0x0); + + pointerEvent.resetHistory(); + this.clock.tick(1500); + + gestureStart('onetap', 15, 20); + gestureEnd('onetap', 15, 20); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 15, 20, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 15, 20, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 15, 20, 0x0); + + pointerEvent.resetHistory(); + }); + }); + + describe('Gesture twotap', function () { + it('should handle gesture twotap events', function () { + let bmask = 0x4; + + gestureStart("twotap", 20, 40); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should keep same position for multiple twotap events', function () { + let bmask = 0x4; + + for (let offset = 0;offset < 30;offset += 10) { + pointerEvent.resetHistory(); + + gestureStart('twotap', 20, 40 + offset); + gestureEnd('twotap', 20, 40 + offset); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + } + }); + }); + + describe('Gesture threetap', function () { + it('should handle gesture start for threetap events', function () { + let bmask = 0x2; + + gestureStart("threetap", 20, 40); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should keep same position for multiple threetap events', function () { + let bmask = 0x2; + + for (let offset = 0;offset < 30;offset += 10) { + pointerEvent.resetHistory(); + + gestureStart('threetap', 20, 40 + offset); + gestureEnd('threetap', 20, 40 + offset); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + } + }); + }); + + describe('Gesture drag', function () { + it('should handle gesture drag events', function () { + let bmask = 0x1; + + gestureStart('drag', 20, 40); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + + pointerEvent.resetHistory(); + + gestureMove('drag', 30, 50); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledOnce; + expect(pointerEvent).to.have.been.calledWith(client._sock, + 30, 50, bmask); + + pointerEvent.resetHistory(); + + gestureEnd('drag', 30, 50); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 30, 50, bmask); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 30, 50, 0x0); + }); + }); + + describe('Gesture long press', function () { + it('should handle long press events', function () { + let bmask = 0x4; + + gestureStart('longpress', 20, 40); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + pointerEvent.resetHistory(); + + gestureMove('longpress', 40, 60); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 40, 60, bmask); + + pointerEvent.resetHistory(); + + gestureEnd('longpress', 40, 60); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 40, 60, bmask); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 40, 60, 0x0); + }); + }); + + describe('Gesture twodrag', function () { + it('should handle gesture twodrag up events', function () { + let bmask = 0x10; // Button mask for scroll down + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 0, -60); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle gesture twodrag down events', function () { + let bmask = 0x8; // Button mask for scroll up + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 0, 60); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle gesture twodrag right events', function () { + let bmask = 0x20; // Button mask for scroll right + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 60, 0); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle gesture twodrag left events', function () { + let bmask = 0x40; // Button mask for scroll left + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, -60, 0); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle gesture twodrag diag events', function () { + let scrlUp = 0x8; // Button mask for scroll up + let scrlRight = 0x20; // Button mask for scroll right + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 60, 60); + + expect(pointerEvent).to.have.been.callCount(5); + expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.getCall(1)).to.have.been.calledWith(client._sock, + 20, 40, scrlUp); + expect(pointerEvent.getCall(2)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.getCall(3)).to.have.been.calledWith(client._sock, + 20, 40, scrlRight); + expect(pointerEvent.getCall(4)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle multiple small gesture twodrag events', function () { + let bmask = 0x8; // Button mask for scroll up + + gestureStart('twodrag', 20, 40, 0, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 0, 10); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 0, 20); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 20, 40, 0, 60); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + }); + + it('should handle large gesture twodrag events', function () { + let bmask = 0x8; // Button mask for scroll up + + gestureStart('twodrag', 30, 50, 0, 0); + + expect(pointerEvent). + to.have.been.calledOnceWith(client._sock, 30, 50, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('twodrag', 30, 50, 0, 200); + + expect(pointerEvent).to.have.callCount(7); + expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, + 30, 50, 0x0); + expect(pointerEvent.getCall(1)).to.have.been.calledWith(client._sock, + 30, 50, bmask); + expect(pointerEvent.getCall(2)).to.have.been.calledWith(client._sock, + 30, 50, 0x0); + expect(pointerEvent.getCall(3)).to.have.been.calledWith(client._sock, + 30, 50, bmask); + expect(pointerEvent.getCall(4)).to.have.been.calledWith(client._sock, + 30, 50, 0x0); + expect(pointerEvent.getCall(5)).to.have.been.calledWith(client._sock, + 30, 50, bmask); + expect(pointerEvent.getCall(6)).to.have.been.calledWith(client._sock, + 30, 50, 0x0); + }); + }); + + describe('Gesture pinch', function () { + let keyEvent; + let qemuKeyEvent; + + beforeEach(function () { + // Touch events and gestures are not supported on IE + if (browser.isIE()) { + this.skip(); + return; + } + + keyEvent = sinon.spy(RFB.messages, 'keyEvent'); + qemuKeyEvent = sinon.spy(RFB.messages, 'QEMUExtendedKeyEvent'); + }); + + afterEach(function () { + keyEvent.restore(); + qemuKeyEvent.restore(); + }); + + it('should handle gesture pinch in events', function () { + let keysym = KeyTable.XK_Control_L; + let bmask = 0x10; // Button mask for scroll down + + gestureStart('pinch', 20, 40, 90, 90); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + expect(keyEvent).to.not.have.been.called; + + pointerEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 30, 30); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + expect(keyEvent).to.have.been.calledTwice; + expect(keyEvent.firstCall).to.have.been.calledWith(client._sock, + keysym, 1); + expect(keyEvent.secondCall).to.have.been.calledWith(client._sock, + keysym, 0); + + expect(keyEvent.firstCall).to.have.been.calledBefore(pointerEvent.secondCall); + expect(keyEvent.lastCall).to.have.been.calledAfter(pointerEvent.lastCall); + + pointerEvent.resetHistory(); + keyEvent.resetHistory(); + + gestureEnd('pinch', 20, 40); + + expect(pointerEvent).to.not.have.been.called; + expect(keyEvent).to.not.have.been.called; + }); + + it('should handle gesture pinch out events', function () { + let keysym = KeyTable.XK_Control_L; + let bmask = 0x8; // Button mask for scroll up + + gestureStart('pinch', 10, 20, 10, 20); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 20, 0x0); + expect(keyEvent).to.not.have.been.called; + + pointerEvent.resetHistory(); + + gestureMove('pinch', 10, 20, 70, 80); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 20, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 20, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 10, 20, 0x0); + + expect(keyEvent).to.have.been.calledTwice; + expect(keyEvent.firstCall).to.have.been.calledWith(client._sock, + keysym, 1); + expect(keyEvent.secondCall).to.have.been.calledWith(client._sock, + keysym, 0); + + expect(keyEvent.firstCall).to.have.been.calledBefore(pointerEvent.secondCall); + expect(keyEvent.lastCall).to.have.been.calledAfter(pointerEvent.lastCall); + + pointerEvent.resetHistory(); + keyEvent.resetHistory(); + + gestureEnd('pinch', 10, 20); + + expect(pointerEvent).to.not.have.been.called; + expect(keyEvent).to.not.have.been.called; + }); + + it('should handle large gesture pinch', function () { + let keysym = KeyTable.XK_Control_L; + let bmask = 0x10; // Button mask for scroll down + + gestureStart('pinch', 20, 40, 150, 150); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + expect(keyEvent).to.not.have.been.called; + + pointerEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 30, 30); + + expect(pointerEvent).to.have.been.callCount(5); + expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.getCall(1)).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.getCall(2)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.getCall(3)).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.getCall(4)).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + expect(keyEvent).to.have.been.calledTwice; + expect(keyEvent.firstCall).to.have.been.calledWith(client._sock, + keysym, 1); + expect(keyEvent.secondCall).to.have.been.calledWith(client._sock, + keysym, 0); + + expect(keyEvent.firstCall).to.have.been.calledBefore(pointerEvent.secondCall); + expect(keyEvent.lastCall).to.have.been.calledAfter(pointerEvent.lastCall); + + pointerEvent.resetHistory(); + keyEvent.resetHistory(); + + gestureEnd('pinch', 20, 40); + + expect(pointerEvent).to.not.have.been.called; + expect(keyEvent).to.not.have.been.called; + }); + + it('should handle multiple small gesture pinch out events', function () { + let keysym = KeyTable.XK_Control_L; + let bmask = 0x8; // Button mask for scroll down + + gestureStart('pinch', 20, 40, 0, 10); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + expect(keyEvent).to.not.have.been.called; + + pointerEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 0, 30); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 0, 60); + clock.tick(50); + + expect(pointerEvent).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + pointerEvent.resetHistory(); + keyEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 0, 90); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + expect(keyEvent).to.have.been.calledTwice; + expect(keyEvent.firstCall).to.have.been.calledWith(client._sock, + keysym, 1); + expect(keyEvent.secondCall).to.have.been.calledWith(client._sock, + keysym, 0); + + expect(keyEvent.firstCall).to.have.been.calledBefore(pointerEvent.secondCall); + expect(keyEvent.lastCall).to.have.been.calledAfter(pointerEvent.lastCall); + + pointerEvent.resetHistory(); + keyEvent.resetHistory(); + + gestureEnd('pinch', 20, 40); + + expect(keyEvent).to.not.have.been.called; + }); + + it('should send correct key control code', function () { + let keysym = KeyTable.XK_Control_L; + let code = 0x1d; + let bmask = 0x10; // Button mask for scroll down + + client._qemuExtKeyEventSupported = true; + + gestureStart('pinch', 20, 40, 90, 90); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 40, 0x0); + expect(qemuKeyEvent).to.not.have.been.called; + + pointerEvent.resetHistory(); + + gestureMove('pinch', 20, 40, 30, 30); + + expect(pointerEvent).to.have.been.calledThrice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 40, bmask); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 20, 40, 0x0); + + expect(qemuKeyEvent).to.have.been.calledTwice; + expect(qemuKeyEvent.firstCall).to.have.been.calledWith(client._sock, + keysym, + true, + code); + expect(qemuKeyEvent.secondCall).to.have.been.calledWith(client._sock, + keysym, + false, + code); + + expect(qemuKeyEvent.firstCall).to.have.been.calledBefore(pointerEvent.secondCall); + expect(qemuKeyEvent.lastCall).to.have.been.calledAfter(pointerEvent.lastCall); + + pointerEvent.resetHistory(); + qemuKeyEvent.resetHistory(); + + gestureEnd('pinch', 20, 40); + + expect(pointerEvent).to.not.have.been.called; + expect(qemuKeyEvent).to.not.have.been.called; + }); + }); + }); + + describe('WebSocket Events', function () { // message events it('should do nothing if we receive an empty message and have nothing in the queue', function () { client._normalMsg = sinon.spy(); diff --git a/vnc.html b/vnc.html index 0f2a3b35..32f356f3 100644 --- a/vnc.html +++ b/vnc.html @@ -94,18 +94,6 @@
- - - -
From 07a69954b17ee1fecaf39be876b68a55ad07ba1b Mon Sep 17 00:00:00 2001 From: Samuel Mannehed Date: Wed, 10 Jun 2020 10:23:50 +0200 Subject: [PATCH 05/11] Add lint rule for function declaration indentation --- .eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc b/.eslintrc index 4b50d2ff..a53bb402 100644 --- a/.eslintrc +++ b/.eslintrc @@ -25,6 +25,7 @@ "brace-style": ["error", "1tbs", { "allowSingleLine": true }], "indent": ["error", 4, { "SwitchCase": 1, + "FunctionDeclaration": { "parameters": "first" }, "CallExpression": { "arguments": "first" }, "ArrayExpression": "first", "ObjectExpression": "first", From 77c32d164d19c42d0bb03992cf85899532d80de1 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 10 Jun 2020 12:55:27 +0200 Subject: [PATCH 06/11] Remove delayed wheel timer This isn't really expected behaviour from a user, i.e. that an extremely small wheel movement still gives a large scroll event in the remote application. --- core/input/mouse.js | 22 ---------------------- tests/test.mouse.js | 22 ---------------------- 2 files changed, 44 deletions(-) diff --git a/core/input/mouse.js b/core/input/mouse.js index 82ea335a..4d015669 100644 --- a/core/input/mouse.js +++ b/core/input/mouse.js @@ -8,7 +8,6 @@ import * as Log from '../util/logging.js'; import { setCapture, stopEvent, getPointerEvent } from '../util/events.js'; const WHEEL_STEP = 10; // Delta threshold for a mouse wheel step -const WHEEL_STEP_TIMEOUT = 50; // ms const WHEEL_LINE_HEIGHT = 19; export default class Mouse { @@ -16,8 +15,6 @@ export default class Mouse { this._target = target || document; this._pos = null; - this._wheelStepXTimer = null; - this._wheelStepYTimer = null; this._accumulatedWheelDeltaX = 0; this._accumulatedWheelDeltaY = 0; @@ -102,16 +99,7 @@ export default class Mouse { this._accumulatedWheelDeltaY = 0; } - _resetWheelStepTimers() { - window.clearTimeout(this._wheelStepXTimer); - window.clearTimeout(this._wheelStepYTimer); - this._wheelStepXTimer = null; - this._wheelStepYTimer = null; - } - _handleMouseWheel(e) { - this._resetWheelStepTimers(); - this._updateMousePosition(e); let dX = e.deltaX; @@ -136,17 +124,9 @@ export default class Mouse { // after a timeout. if (Math.abs(this._accumulatedWheelDeltaX) > WHEEL_STEP) { this._generateWheelStepX(); - } else { - this._wheelStepXTimer = - window.setTimeout(this._generateWheelStepX.bind(this), - WHEEL_STEP_TIMEOUT); } if (Math.abs(this._accumulatedWheelDeltaY) > WHEEL_STEP) { this._generateWheelStepY(); - } else { - this._wheelStepYTimer = - window.setTimeout(this._generateWheelStepY.bind(this), - WHEEL_STEP_TIMEOUT); } stopEvent(e); @@ -214,8 +194,6 @@ export default class Mouse { ungrab() { const t = this._target; - this._resetWheelStepTimers(); - t.removeEventListener('mousedown', this._eventHandlers.mousedown); t.removeEventListener('mouseup', this._eventHandlers.mouseup); t.removeEventListener('mousemove', this._eventHandlers.mousemove); diff --git a/tests/test.mouse.js b/tests/test.mouse.js index 8830c4be..9e06ff26 100644 --- a/tests/test.mouse.js +++ b/tests/test.mouse.js @@ -90,9 +90,6 @@ describe('Mouse Event Handling', function () { describe('Accumulate mouse wheel events with small delta', function () { - beforeEach(function () { this.clock = sinon.useFakeTimers(); }); - afterEach(function () { this.clock.restore(); }); - it('should accumulate wheel events if small enough', function () { const mouse = new Mouse(target); mouse.onmousebutton = sinon.spy(); @@ -100,7 +97,6 @@ describe('Mouse Event Handling', function () { mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 4, deltaY: 0, deltaMode: 0 })); - this.clock.tick(10); mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 4, deltaY: 0, deltaMode: 0 })); @@ -108,14 +104,12 @@ describe('Mouse Event Handling', function () { // threshold is 10 expect(mouse._accumulatedWheelDeltaX).to.be.equal(8); - this.clock.tick(10); mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 4, deltaY: 0, deltaMode: 0 })); expect(mouse.onmousebutton).to.have.callCount(2); // mouse down and up - this.clock.tick(10); mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 4, deltaY: 9, deltaMode: 0 })); @@ -133,11 +127,9 @@ describe('Mouse Event Handling', function () { mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 11, deltaY: 0, deltaMode: 0 })); - this.clock.tick(10); mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 0, deltaY: 70, deltaMode: 0 })); - this.clock.tick(10); mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 400, deltaY: 400, deltaMode: 0 })); @@ -145,18 +137,6 @@ describe('Mouse Event Handling', function () { expect(mouse.onmousebutton).to.have.callCount(8); // mouse down and up }); - it('should send even small wheel events after a timeout', function () { - const mouse = new Mouse(target); - mouse.onmousebutton = sinon.spy(); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 1, deltaY: 0, deltaMode: 0 })); - this.clock.tick(51); // timeout on 50 ms - - expect(mouse.onmousebutton).to.have.callCount(2); // mouse down and up - }); - it('should account for non-zero deltaMode', function () { const mouse = new Mouse(target); mouse.onmousebutton = sinon.spy(); @@ -165,8 +145,6 @@ describe('Mouse Event Handling', function () { 'mousewheel', { clientX: 18, clientY: 40, deltaX: 0, deltaY: 2, deltaMode: 1 })); - this.clock.tick(10); - mouse._handleMouseWheel(mouseevent( 'mousewheel', { clientX: 18, clientY: 40, deltaX: 1, deltaY: 0, deltaMode: 2 })); From 4a87038080ee8d9ad8eb9f276fc6acde6ef87467 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 10 Jun 2020 12:56:50 +0200 Subject: [PATCH 07/11] Remove very legacy mouse event handling This is only needed on such ancient versions of Internet Explorer that it wouldn't satisfy our other requirements anyway. --- core/input/mouse.js | 11 +---------- tests/test.mouse.js | 6 +++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/core/input/mouse.js b/core/input/mouse.js index 4d015669..8c917d09 100644 --- a/core/input/mouse.js +++ b/core/input/mouse.js @@ -42,16 +42,7 @@ export default class Mouse { this._updateMousePosition(e); let pos = this._pos; - let bmask; - if (e.which) { - /* everything except IE */ - bmask = 1 << e.button; - } else { - /* IE including 9 */ - bmask = (e.button & 0x1) + // Left - (e.button & 0x2) * 2 + // Right - (e.button & 0x4) / 2; // Middle - } + let bmask = 1 << e.button; Log.Debug("onmousebutton " + (down ? "down" : "up") + ", x: " + pos.x + ", y: " + pos.y + ", bmask: " + bmask); diff --git a/tests/test.mouse.js b/tests/test.mouse.js index 9e06ff26..13bd0c60 100644 --- a/tests/test.mouse.js +++ b/tests/test.mouse.js @@ -43,7 +43,7 @@ describe('Mouse Event Handling', function () { expect(down).to.be.equal(1); done(); }; - mouse._handleMouseDown(mouseevent('mousedown', { button: '0x01' })); + mouse._handleMouseDown(mouseevent('mousedown', { button: 0 })); }); it('should decode mouseup events', function (done) { let calls = 0; @@ -55,8 +55,8 @@ describe('Mouse Event Handling', function () { done(); } }; - mouse._handleMouseDown(mouseevent('mousedown', { button: '0x01' })); - mouse._handleMouseUp(mouseevent('mouseup', { button: '0x01' })); + mouse._handleMouseDown(mouseevent('mousedown', { button: 0 })); + mouse._handleMouseUp(mouseevent('mouseup', { button: 0 })); }); it('should decode mousemove events', function (done) { const mouse = new Mouse(target); From f84bc57bdaf6ed67804fee7d9caadd94c65d1bac Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 10 Jun 2020 13:59:10 +0200 Subject: [PATCH 08/11] Move wheel event handling to RFB class The Mouse class does very little now so it mostly just obfuscate things. Move everything directly in to the RFB class instead. --- core/input/mouse.js | 70 --------------- core/rfb.js | 66 ++++++++++++++ tests/test.mouse.js | 82 ----------------- tests/test.rfb.js | 212 ++++++++++++++++++++++++++++++++------------ 4 files changed, 221 insertions(+), 209 deletions(-) diff --git a/core/input/mouse.js b/core/input/mouse.js index 8c917d09..794adfe2 100644 --- a/core/input/mouse.js +++ b/core/input/mouse.js @@ -7,22 +7,16 @@ import * as Log from '../util/logging.js'; import { setCapture, stopEvent, getPointerEvent } from '../util/events.js'; -const WHEEL_STEP = 10; // Delta threshold for a mouse wheel step -const WHEEL_LINE_HEIGHT = 19; - export default class Mouse { constructor(target) { this._target = target || document; this._pos = null; - this._accumulatedWheelDeltaX = 0; - this._accumulatedWheelDeltaY = 0; this._eventHandlers = { 'mousedown': this._handleMouseDown.bind(this), 'mouseup': this._handleMouseUp.bind(this), 'mousemove': this._handleMouseMove.bind(this), - 'mousewheel': this._handleMouseWheel.bind(this), 'mousedisable': this._handleMouseDisable.bind(this) }; @@ -61,68 +55,6 @@ export default class Mouse { this._handleMouseButton(e, 0); } - // Mouse wheel events are sent in steps over VNC. This means that the VNC - // protocol can't handle a wheel event with specific distance or speed. - // Therefor, if we get a lot of small mouse wheel events we combine them. - _generateWheelStepX() { - - if (this._accumulatedWheelDeltaX < 0) { - this.onmousebutton(this._pos.x, this._pos.y, 1, 1 << 5); - this.onmousebutton(this._pos.x, this._pos.y, 0, 1 << 5); - } else if (this._accumulatedWheelDeltaX > 0) { - this.onmousebutton(this._pos.x, this._pos.y, 1, 1 << 6); - this.onmousebutton(this._pos.x, this._pos.y, 0, 1 << 6); - } - - this._accumulatedWheelDeltaX = 0; - } - - _generateWheelStepY() { - - if (this._accumulatedWheelDeltaY < 0) { - this.onmousebutton(this._pos.x, this._pos.y, 1, 1 << 3); - this.onmousebutton(this._pos.x, this._pos.y, 0, 1 << 3); - } else if (this._accumulatedWheelDeltaY > 0) { - this.onmousebutton(this._pos.x, this._pos.y, 1, 1 << 4); - this.onmousebutton(this._pos.x, this._pos.y, 0, 1 << 4); - } - - this._accumulatedWheelDeltaY = 0; - } - - _handleMouseWheel(e) { - this._updateMousePosition(e); - - let dX = e.deltaX; - let dY = e.deltaY; - - // Pixel units unless it's non-zero. - // Note that if deltamode is line or page won't matter since we aren't - // sending the mouse wheel delta to the server anyway. - // The difference between pixel and line can be important however since - // we have a threshold that can be smaller than the line height. - if (e.deltaMode !== 0) { - dX *= WHEEL_LINE_HEIGHT; - dY *= WHEEL_LINE_HEIGHT; - } - - this._accumulatedWheelDeltaX += dX; - this._accumulatedWheelDeltaY += dY; - - // Generate a mouse wheel step event when the accumulated delta - // for one of the axes is large enough. - // Small delta events that do not pass the threshold get sent - // after a timeout. - if (Math.abs(this._accumulatedWheelDeltaX) > WHEEL_STEP) { - this._generateWheelStepX(); - } - if (Math.abs(this._accumulatedWheelDeltaY) > WHEEL_STEP) { - this._generateWheelStepY(); - } - - stopEvent(e); - } - _handleMouseMove(e) { this._updateMousePosition(e); this.onmousemove(this._pos.x, this._pos.y); @@ -172,7 +104,6 @@ export default class Mouse { t.addEventListener('mousedown', this._eventHandlers.mousedown); t.addEventListener('mouseup', this._eventHandlers.mouseup); t.addEventListener('mousemove', this._eventHandlers.mousemove); - t.addEventListener('wheel', this._eventHandlers.mousewheel); // Prevent middle-click pasting (see above for why we bind to document) document.addEventListener('click', this._eventHandlers.mousedisable); @@ -188,7 +119,6 @@ export default class Mouse { t.removeEventListener('mousedown', this._eventHandlers.mousedown); t.removeEventListener('mouseup', this._eventHandlers.mouseup); t.removeEventListener('mousemove', this._eventHandlers.mousemove); - t.removeEventListener('wheel', this._eventHandlers.mousewheel); document.removeEventListener('click', this._eventHandlers.mousedisable); diff --git a/core/rfb.js b/core/rfb.js index 9db375fc..4d1dcf1e 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -41,6 +41,10 @@ const DEFAULT_BACKGROUND = 'rgb(40, 40, 40)'; // Minimum wait (ms) between two mouse moves const MOUSE_MOVE_DELAY = 17; +// Wheel thresholds +const WHEEL_STEP = 10; // Pixels needed for one step +const WHEEL_LINE_HEIGHT = 19; // Assumed pixels for one line step + // Gesture thresholds const GESTURE_ZOOMSENS = 75; const GESTURE_SCRLSENS = 50; @@ -152,6 +156,8 @@ export default class RFB extends EventTargetMixin { this._viewportDragging = false; this._viewportDragPos = {}; this._viewportHasMoved = false; + this._accumulatedWheelDeltaX = 0; + this._accumulatedWheelDeltaY = 0; // Gesture state this._gestureLastTapTime = null; @@ -163,6 +169,7 @@ export default class RFB extends EventTargetMixin { this._eventHandlers = { focusCanvas: this._focusCanvas.bind(this), windowResize: this._windowResize.bind(this), + handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), }; @@ -532,6 +539,9 @@ export default class RFB extends EventTargetMixin { this._canvas.addEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.addEventListener("touchstart", this._eventHandlers.focusCanvas); + // Wheel events + this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); + // Gesture events this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture); this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture); @@ -546,6 +556,7 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture); this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture); this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture); + this._canvas.removeEventListener("wheel", this._eventHandlers.handleWheel); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); @@ -939,6 +950,61 @@ export default class RFB extends EventTargetMixin { this._display.absY(y), mask); } + _handleWheel(ev) { + if (this._rfbConnectionState !== 'connected') { return; } + if (this._viewOnly) { return; } // View only, skip mouse events + + ev.stopPropagation(); + ev.preventDefault(); + + let pos = clientToElement(ev.clientX, ev.clientY, + this._canvas); + + let dX = ev.deltaX; + let dY = ev.deltaY; + + // Pixel units unless it's non-zero. + // Note that if deltamode is line or page won't matter since we aren't + // sending the mouse wheel delta to the server anyway. + // The difference between pixel and line can be important however since + // we have a threshold that can be smaller than the line height. + if (ev.deltaMode !== 0) { + dX *= WHEEL_LINE_HEIGHT; + dY *= WHEEL_LINE_HEIGHT; + } + + // Mouse wheel events are sent in steps over VNC. This means that the VNC + // protocol can't handle a wheel event with specific distance or speed. + // Therefor, if we get a lot of small mouse wheel events we combine them. + this._accumulatedWheelDeltaX += dX; + this._accumulatedWheelDeltaY += dY; + + // Generate a mouse wheel step event when the accumulated delta + // for one of the axes is large enough. + if (Math.abs(this._accumulatedWheelDeltaX) > WHEEL_STEP) { + if (this._accumulatedWheelDeltaX < 0) { + this._handleMouseButton(pos.x, pos.y, true, 1 << 5); + this._handleMouseButton(pos.x, pos.y, false, 1 << 5); + } else if (this._accumulatedWheelDeltaX > 0) { + this._handleMouseButton(pos.x, pos.y, true, 1 << 6); + this._handleMouseButton(pos.x, pos.y, false, 1 << 6); + } + + this._accumulatedWheelDeltaX = 0; + } + if (Math.abs(this._accumulatedWheelDeltaY) > WHEEL_STEP) { + if (this._accumulatedWheelDeltaY < 0) { + this._handleMouseButton(pos.x, pos.y, true, 1 << 3); + this._handleMouseButton(pos.x, pos.y, false, 1 << 3); + } else if (this._accumulatedWheelDeltaY > 0) { + this._handleMouseButton(pos.x, pos.y, true, 1 << 4); + this._handleMouseButton(pos.x, pos.y, false, 1 << 4); + } + + this._accumulatedWheelDeltaY = 0; + } + } + _handleTapEvent(ev, bmask) { let pos = clientToElement(ev.detail.clientX, ev.detail.clientY, this._canvas); diff --git a/tests/test.mouse.js b/tests/test.mouse.js index 13bd0c60..25d52195 100644 --- a/tests/test.mouse.js +++ b/tests/test.mouse.js @@ -69,87 +69,5 @@ describe('Mouse Event Handling', function () { mouse._handleMouseMove(mouseevent('mousemove', { clientX: 50, clientY: 20 })); }); - it('should decode mousewheel events', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - calls++; - expect(bmask).to.be.equal(1<<6); - if (calls === 1) { - expect(down).to.be.equal(1); - } else if (calls === 2) { - expect(down).to.not.be.equal(1); - done(); - } - }; - mouse._handleMouseWheel(mouseevent('mousewheel', - { deltaX: 50, deltaY: 0, - deltaMode: 0})); - }); - }); - - describe('Accumulate mouse wheel events with small delta', function () { - - it('should accumulate wheel events if small enough', function () { - const mouse = new Mouse(target); - mouse.onmousebutton = sinon.spy(); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 4, deltaY: 0, deltaMode: 0 })); - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 4, deltaY: 0, deltaMode: 0 })); - - // threshold is 10 - expect(mouse._accumulatedWheelDeltaX).to.be.equal(8); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 4, deltaY: 0, deltaMode: 0 })); - - expect(mouse.onmousebutton).to.have.callCount(2); // mouse down and up - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 4, deltaY: 9, deltaMode: 0 })); - - expect(mouse._accumulatedWheelDeltaX).to.be.equal(4); - expect(mouse._accumulatedWheelDeltaY).to.be.equal(9); - - expect(mouse.onmousebutton).to.have.callCount(2); // still - }); - - it('should not accumulate large wheel events', function () { - const mouse = new Mouse(target); - mouse.onmousebutton = sinon.spy(); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 11, deltaY: 0, deltaMode: 0 })); - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 0, deltaY: 70, deltaMode: 0 })); - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 400, deltaY: 400, deltaMode: 0 })); - - expect(mouse.onmousebutton).to.have.callCount(8); // mouse down and up - }); - - it('should account for non-zero deltaMode', function () { - const mouse = new Mouse(target); - mouse.onmousebutton = sinon.spy(); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 0, deltaY: 2, deltaMode: 1 })); - - mouse._handleMouseWheel(mouseevent( - 'mousewheel', { clientX: 18, clientY: 40, - deltaX: 1, deltaY: 0, deltaMode: 2 })); - - expect(mouse.onmousebutton).to.have.callCount(4); // mouse down and up - }); }); }); diff --git a/tests/test.rfb.js b/tests/test.rfb.js index e46f40c5..909b300a 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2731,20 +2731,44 @@ describe('Remote Frame Buffer Protocol Client', function () { describe('Asynchronous Events', function () { let client; + let pointerEvent; + let keyEvent; + let qemuKeyEvent; + beforeEach(function () { client = makeRFB(); + client._display.resize(100, 100); + + pointerEvent = sinon.spy(RFB.messages, 'pointerEvent'); + keyEvent = sinon.spy(RFB.messages, 'keyEvent'); + qemuKeyEvent = sinon.spy(RFB.messages, 'QEMUExtendedKeyEvent'); }); - describe('Mouse event handlers', function () { - beforeEach(function () { - this.clock = sinon.useFakeTimers(Date.now()); - sinon.spy(RFB.messages, 'pointerEvent'); - }); - afterEach(function () { - this.clock.restore(); - RFB.messages.pointerEvent.restore(); - }); + afterEach(function () { + pointerEvent.restore(); + keyEvent.restore(); + qemuKeyEvent.restore(); + }); + function elementToClient(x, y) { + let res = { x: 0, y: 0 }; + + let bounds = client._canvas.getBoundingClientRect(); + + /* + * If the canvas is on a fractional position we will calculate + * a fractional mouse position. But that gets truncated when we + * send the event, AND the same thing happens in RFB when it + * generates the PointerEvent message. To compensate for that + * fact we round the value upwards here. + */ + res.x = Math.ceil(bounds.left + x); + res.y = Math.ceil(bounds.top + y); + + return res; + } + + describe('Mouse Events', function () { it('should not send button messages in view-only mode', function () { client._viewOnly = true; client._handleMouseButton(0, 0, 1, 0x001); @@ -2878,7 +2902,128 @@ describe('Remote Frame Buffer Protocol Client', function () { }); }); - describe('Keyboard Event Handlers', function () { + describe('Wheel Events', function () { + function sendWheelEvent(x, y, dx, dy, mode=0) { + let pos = elementToClient(x, y); + let ev; + + try { + ev = new WheelEvent('wheel', + { 'screenX': pos.x + window.screenX, + 'screenY': pos.y + window.screenY, + 'clientX': pos.x, + 'clientY': pos.y, + 'deltaX': dx, + 'deltaY': dy, + 'deltaMode': mode }); + } catch (e) { + ev = document.createEvent('WheelEvent'); + ev.initWheelEvent('wheel', true, true, window, 0, + pos.x + window.screenX, + pos.y + window.screenY, + pos.x, pos.y, + 0, null, "", + dx, dy, 0, mode); + } + + client._canvas.dispatchEvent(ev); + } + + it('should handle wheel up event', function () { + sendWheelEvent(10, 10, 0, -50); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<3); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should handle wheel down event', function () { + sendWheelEvent(10, 10, 0, 50); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should handle wheel left event', function () { + sendWheelEvent(10, 10, -50, 0); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<5); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should handle wheel right event', function () { + sendWheelEvent(10, 10, 50, 0); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<6); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should ignore wheel when in view only', function () { + client._viewOnly = true; + + sendWheelEvent(10, 10, 50, 0); + + expect(pointerEvent).to.not.have.been.called; + }); + + it('should accumulate wheel events if small enough', function () { + sendWheelEvent(10, 10, 0, 4); + sendWheelEvent(10, 10, 0, 4); + + expect(pointerEvent).to.not.have.been.called; + + sendWheelEvent(10, 10, 0, 4); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should not accumulate large wheel events', function () { + sendWheelEvent(10, 10, 0, 400); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should handle line based wheel event', function () { + sendWheelEvent(10, 10, 0, 1, 1); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + + it('should handle page based wheel event', function () { + sendWheelEvent(10, 10, 0, 1, 2); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 1<<4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0); + }); + }); + + describe('Keyboard Events', function () { it('should send a key message on a key press', function () { client._handleKeyEvent(0x41, 'KeyA', true); const keyMsg = {_sQ: new Uint8Array(8), _sQlen: 0, flush: () => {}}; @@ -2895,42 +3040,14 @@ describe('Remote Frame Buffer Protocol Client', function () { }); describe('Gesture event handlers', function () { - let pointerEvent; - beforeEach(function () { // Touch events and gestures are not supported on IE if (browser.isIE()) { this.skip(); return; } - - pointerEvent = sinon.spy(RFB.messages, 'pointerEvent'); - - client._display.resize(100, 100); }); - afterEach(function () { - pointerEvent.restore(); - }); - - function elementToClient(x, y) { - let res = { x: 0, y: 0 }; - - let bounds = client._canvas.getBoundingClientRect(); - - /* - * If the canvas is on a fractional position we will calculate - * a fractional mouse position. But that gets truncated when we - * send the event, AND the same thing happens in RFB when it - * generates the PointerEvent message. To compensate for that - * fact we round the value upwards here. - */ - res.x = Math.ceil(bounds.left + x); - res.y = Math.ceil(bounds.top + y); - - return res; - } - function gestureStart(gestureType, x, y, magnitudeX = 0, magnitudeY = 0) { let pos = elementToClient(x, y); @@ -3392,25 +3509,6 @@ describe('Remote Frame Buffer Protocol Client', function () { }); describe('Gesture pinch', function () { - let keyEvent; - let qemuKeyEvent; - - beforeEach(function () { - // Touch events and gestures are not supported on IE - if (browser.isIE()) { - this.skip(); - return; - } - - keyEvent = sinon.spy(RFB.messages, 'keyEvent'); - qemuKeyEvent = sinon.spy(RFB.messages, 'QEMUExtendedKeyEvent'); - }); - - afterEach(function () { - keyEvent.restore(); - qemuKeyEvent.restore(); - }); - it('should handle gesture pinch in events', function () { let keysym = KeyTable.XK_Control_L; let bmask = 0x10; // Button mask for scroll down From 88589a44f7ff8bdcb9f66bc9160e83a6f0827bb5 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 10 Jun 2020 14:12:22 +0200 Subject: [PATCH 09/11] Increase wheel step threshold The previous value made the detection too sensitive and it was very difficult to scroll precisely. A value of 50 pixels should give similar behaviour to systems that don't do fine grained scrolling. --- core/rfb.js | 6 +++--- tests/test.rfb.js | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 4d1dcf1e..c5b60160 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -42,7 +42,7 @@ const DEFAULT_BACKGROUND = 'rgb(40, 40, 40)'; const MOUSE_MOVE_DELAY = 17; // Wheel thresholds -const WHEEL_STEP = 10; // Pixels needed for one step +const WHEEL_STEP = 50; // Pixels needed for one step const WHEEL_LINE_HEIGHT = 19; // Assumed pixels for one line step // Gesture thresholds @@ -981,7 +981,7 @@ export default class RFB extends EventTargetMixin { // Generate a mouse wheel step event when the accumulated delta // for one of the axes is large enough. - if (Math.abs(this._accumulatedWheelDeltaX) > WHEEL_STEP) { + if (Math.abs(this._accumulatedWheelDeltaX) >= WHEEL_STEP) { if (this._accumulatedWheelDeltaX < 0) { this._handleMouseButton(pos.x, pos.y, true, 1 << 5); this._handleMouseButton(pos.x, pos.y, false, 1 << 5); @@ -992,7 +992,7 @@ export default class RFB extends EventTargetMixin { this._accumulatedWheelDeltaX = 0; } - if (Math.abs(this._accumulatedWheelDeltaY) > WHEEL_STEP) { + if (Math.abs(this._accumulatedWheelDeltaY) >= WHEEL_STEP) { if (this._accumulatedWheelDeltaY < 0) { this._handleMouseButton(pos.x, pos.y, true, 1 << 3); this._handleMouseButton(pos.x, pos.y, false, 1 << 3); diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 909b300a..cc450cd9 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2978,12 +2978,12 @@ describe('Remote Frame Buffer Protocol Client', function () { }); it('should accumulate wheel events if small enough', function () { - sendWheelEvent(10, 10, 0, 4); - sendWheelEvent(10, 10, 0, 4); + sendWheelEvent(10, 10, 0, 20); + sendWheelEvent(10, 10, 0, 20); expect(pointerEvent).to.not.have.been.called; - sendWheelEvent(10, 10, 0, 4); + sendWheelEvent(10, 10, 0, 20); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3003,7 +3003,7 @@ describe('Remote Frame Buffer Protocol Client', function () { }); it('should handle line based wheel event', function () { - sendWheelEvent(10, 10, 0, 1, 1); + sendWheelEvent(10, 10, 0, 3, 1); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3013,7 +3013,7 @@ describe('Remote Frame Buffer Protocol Client', function () { }); it('should handle page based wheel event', function () { - sendWheelEvent(10, 10, 0, 1, 2); + sendWheelEvent(10, 10, 0, 3, 2); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, From 50cde2faab54e1bfcf411ab3fb5c2950f444e81f Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 10 Jun 2020 16:13:03 +0200 Subject: [PATCH 10/11] Move mouse event handling to RFB class Move the last remaining bits to the RFB class to keep things simple, as the Mouse class no longer provides any real value. --- core/input/mouse.js | 127 --------------- core/rfb.js | 72 +++++++-- docs/API-internal.md | 40 +---- tests/test.mouse.js | 73 --------- tests/test.rfb.js | 364 ++++++++++++++++++++++++++++++------------- 5 files changed, 324 insertions(+), 352 deletions(-) delete mode 100644 core/input/mouse.js delete mode 100644 tests/test.mouse.js diff --git a/core/input/mouse.js b/core/input/mouse.js deleted file mode 100644 index 794adfe2..00000000 --- a/core/input/mouse.js +++ /dev/null @@ -1,127 +0,0 @@ -/* - * noVNC: HTML5 VNC client - * Copyright (C) 2019 The noVNC Authors - * Licensed under MPL 2.0 or any later version (see LICENSE.txt) - */ - -import * as Log from '../util/logging.js'; -import { setCapture, stopEvent, getPointerEvent } from '../util/events.js'; - -export default class Mouse { - constructor(target) { - this._target = target || document; - - this._pos = null; - - this._eventHandlers = { - 'mousedown': this._handleMouseDown.bind(this), - 'mouseup': this._handleMouseUp.bind(this), - 'mousemove': this._handleMouseMove.bind(this), - 'mousedisable': this._handleMouseDisable.bind(this) - }; - - // ===== EVENT HANDLERS ===== - - this.onmousebutton = () => {}; // Handler for mouse button press/release - this.onmousemove = () => {}; // Handler for mouse movement - } - - // ===== PRIVATE METHODS ===== - - _resetDoubleClickTimer() { - this._doubleClickTimer = null; - } - - _handleMouseButton(e, down) { - this._updateMousePosition(e); - let pos = this._pos; - - let bmask = 1 << e.button; - - Log.Debug("onmousebutton " + (down ? "down" : "up") + - ", x: " + pos.x + ", y: " + pos.y + ", bmask: " + bmask); - this.onmousebutton(pos.x, pos.y, down, bmask); - - stopEvent(e); - } - - _handleMouseDown(e) { - setCapture(this._target); - - this._handleMouseButton(e, 1); - } - - _handleMouseUp(e) { - this._handleMouseButton(e, 0); - } - - _handleMouseMove(e) { - this._updateMousePosition(e); - this.onmousemove(this._pos.x, this._pos.y); - stopEvent(e); - } - - _handleMouseDisable(e) { - /* - * Stop propagation if inside canvas area - * Note: This is only needed for the 'click' event as it fails - * to fire properly for the target element so we have - * to listen on the document element instead. - */ - if (e.target == this._target) { - stopEvent(e); - } - } - - // Update coordinates relative to target - _updateMousePosition(e) { - e = getPointerEvent(e); - const bounds = this._target.getBoundingClientRect(); - let x; - let y; - // Clip to target bounds - if (e.clientX < bounds.left) { - x = 0; - } else if (e.clientX >= bounds.right) { - x = bounds.width - 1; - } else { - x = e.clientX - bounds.left; - } - if (e.clientY < bounds.top) { - y = 0; - } else if (e.clientY >= bounds.bottom) { - y = bounds.height - 1; - } else { - y = e.clientY - bounds.top; - } - this._pos = {x: x, y: y}; - } - - // ===== PUBLIC METHODS ===== - - grab() { - const t = this._target; - t.addEventListener('mousedown', this._eventHandlers.mousedown); - t.addEventListener('mouseup', this._eventHandlers.mouseup); - t.addEventListener('mousemove', this._eventHandlers.mousemove); - - // Prevent middle-click pasting (see above for why we bind to document) - document.addEventListener('click', this._eventHandlers.mousedisable); - - // preventDefault() on mousedown doesn't stop this event for some - // reason so we have to explicitly block it - t.addEventListener('contextmenu', this._eventHandlers.mousedisable); - } - - ungrab() { - const t = this._target; - - t.removeEventListener('mousedown', this._eventHandlers.mousedown); - t.removeEventListener('mouseup', this._eventHandlers.mouseup); - t.removeEventListener('mousemove', this._eventHandlers.mousemove); - - document.removeEventListener('click', this._eventHandlers.mousedisable); - - t.removeEventListener('contextmenu', this._eventHandlers.mousedisable); - } -} diff --git a/core/rfb.js b/core/rfb.js index c5b60160..c56bdc98 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -12,12 +12,12 @@ import * as Log from './util/logging.js'; import { encodeUTF8, decodeUTF8 } from './util/strings.js'; import { dragThreshold } from './util/browser.js'; import { clientToElement } from './util/element.js'; +import { setCapture } from './util/events.js'; import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Inflator from "./inflator.js"; import Deflator from "./deflator.js"; import Keyboard from "./input/keyboard.js"; -import Mouse from "./input/mouse.js"; import GestureHandler from "./input/gesturehandler.js"; import Cursor from "./util/cursor.js"; import Websock from "./websock.js"; @@ -129,7 +129,6 @@ export default class RFB extends EventTargetMixin { this._display = null; // Display object this._flushing = false; // Display flushing state this._keyboard = null; // Keyboard input handler object - this._mouse = null; // Mouse input handler object this._gestures = null; // Gesture input handler object // Timers @@ -169,6 +168,7 @@ export default class RFB extends EventTargetMixin { this._eventHandlers = { focusCanvas: this._focusCanvas.bind(this), windowResize: this._windowResize.bind(this), + handleMouse: this._handleMouse.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), }; @@ -229,10 +229,6 @@ export default class RFB extends EventTargetMixin { this._keyboard = new Keyboard(this._canvas); this._keyboard.onkeyevent = this._handleKeyEvent.bind(this); - this._mouse = new Mouse(this._canvas); - this._mouse.onmousebutton = this._handleMouseButton.bind(this); - this._mouse.onmousemove = this._handleMouseMove.bind(this); - this._gestures = new GestureHandler(); this._sock = new Websock(); @@ -321,10 +317,8 @@ export default class RFB extends EventTargetMixin { this._rfbConnectionState === "connected") { if (viewOnly) { this._keyboard.ungrab(); - this._mouse.ungrab(); } else { this._keyboard.grab(); - this._mouse.grab(); } } } @@ -539,6 +533,16 @@ export default class RFB extends EventTargetMixin { this._canvas.addEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.addEventListener("touchstart", this._eventHandlers.focusCanvas); + // Mouse events + this._canvas.addEventListener('mousedown', this._eventHandlers.handleMouse); + this._canvas.addEventListener('mouseup', this._eventHandlers.handleMouse); + this._canvas.addEventListener('mousemove', this._eventHandlers.handleMouse); + // Prevent middle-click pasting (see handler for why we bind to document) + this._canvas.addEventListener('click', this._eventHandlers.handleMouse); + // preventDefault() on mousedown doesn't stop this event for some + // reason so we have to explicitly block it + this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse); + // Wheel events this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); @@ -557,11 +561,15 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture); this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture); this._canvas.removeEventListener("wheel", this._eventHandlers.handleWheel); + this._canvas.removeEventListener('mousedown', this._eventHandlers.handleMouse); + this._canvas.removeEventListener('mouseup', this._eventHandlers.handleMouse); + this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse); + this._canvas.removeEventListener('click', this._eventHandlers.handleMouse); + this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); this._keyboard.ungrab(); - this._mouse.ungrab(); this._gestures.detach(); this._sock.close(); try { @@ -859,6 +867,51 @@ export default class RFB extends EventTargetMixin { this.sendKey(keysym, code, down); } + _handleMouse(ev) { + /* + * We don't check connection status or viewOnly here as the + * mouse events might be used to control the viewport + */ + + if (ev.type === 'click') { + /* + * Note: This is only needed for the 'click' event as it fails + * to fire properly for the target element so we have + * to listen on the document element instead. + */ + if (ev.target !== this._canvas) { + return; + } + } + + // FIXME: if we're in view-only and not dragging, + // should we stop events? + ev.stopPropagation(); + ev.preventDefault(); + + if ((ev.type === 'click') || (ev.type === 'contextmenu')) { + return; + } + + let pos = clientToElement(ev.clientX, ev.clientY, + this._canvas); + + switch (ev.type) { + case 'mousedown': + setCapture(this._canvas); + this._handleMouseButton(pos.x, pos.y, + true, 1 << ev.button); + break; + case 'mouseup': + this._handleMouseButton(pos.x, pos.y, + false, 1 << ev.button); + break; + case 'mousemove': + this._handleMouseMove(pos.x, pos.y); + break; + } + } + _handleMouseButton(x, y, down, bmask) { if (this.dragViewport) { if (down && !this._viewportDragging) { @@ -1678,7 +1731,6 @@ export default class RFB extends EventTargetMixin { this._resize(width, height); if (!this._viewOnly) { this._keyboard.grab(); } - if (!this._viewOnly) { this._mouse.grab(); } this._fbDepth = 24; diff --git a/docs/API-internal.md b/docs/API-internal.md index f1519422..cb4cc396 100644 --- a/docs/API-internal.md +++ b/docs/API-internal.md @@ -11,9 +11,6 @@ official external API. ## 1.1 Module List -* __Mouse__ (core/input/mouse.js): Mouse input event handler with -limited touch support. - * __Keyboard__ (core/input/keyboard.js): Keyboard input event handler with non-US keyboard support. Translates keyDown and keyUp events to X11 keysym values. @@ -35,52 +32,29 @@ callback event name, and the callback function. ## 2. Modules -## 2.1 Mouse Module +## 2.1 Keyboard Module ### 2.1.1 Configuration Attributes -| name | type | mode | default | description -| ----------- | ---- | ---- | -------- | ------------ -| touchButton | int | RW | 1 | Button mask (1, 2, 4) for which click to send on touch devices. 0 means ignore clicks. - -### 2.1.2 Methods - -| name | parameters | description -| ------ | ---------- | ------------ -| grab | () | Begin capturing mouse events -| ungrab | () | Stop capturing mouse events - -### 2.1.2 Callbacks - -| name | parameters | description -| ------------- | ------------------- | ------------ -| onmousebutton | (x, y, down, bmask) | Handler for mouse button click/release -| onmousemove | (x, y) | Handler for mouse movement - - -## 2.2 Keyboard Module - -### 2.2.1 Configuration Attributes - None -### 2.2.2 Methods +### 2.1.2 Methods | name | parameters | description | ------ | ---------- | ------------ | grab | () | Begin capturing keyboard events | ungrab | () | Stop capturing keyboard events -### 2.2.3 Callbacks +### 2.1.3 Callbacks | name | parameters | description | ---------- | -------------------- | ------------ | onkeypress | (keysym, code, down) | Handler for key press/release -## 2.3 Display Module +## 2.2 Display Module -### 2.3.1 Configuration Attributes +### 2.2.1 Configuration Attributes | name | type | mode | default | description | ------------ | ----- | ---- | ------- | ------------ @@ -89,7 +63,7 @@ None | width | int | RO | | Display area width | height | int | RO | | Display area height -### 2.3.2 Methods +### 2.2.2 Methods | name | parameters | description | ------------------ | ------------------------------------------------------- | ------------ @@ -113,7 +87,7 @@ None | drawImage | (img, x, y) | Draw image and track damage | autoscale | (containerWidth, containerHeight) | Scale the display -### 2.3.3 Callbacks +### 2.2.3 Callbacks | name | parameters | description | ------- | ---------- | ------------ diff --git a/tests/test.mouse.js b/tests/test.mouse.js deleted file mode 100644 index 25d52195..00000000 --- a/tests/test.mouse.js +++ /dev/null @@ -1,73 +0,0 @@ -const expect = chai.expect; - -import Mouse from '../core/input/mouse.js'; - -describe('Mouse Event Handling', function () { - "use strict"; - - let target; - - beforeEach(function () { - // For these tests we can assume that the canvas is 100x100 - // located at coordinates 10x10 - target = document.createElement('canvas'); - target.style.position = "absolute"; - target.style.top = "10px"; - target.style.left = "10px"; - target.style.width = "100px"; - target.style.height = "100px"; - document.body.appendChild(target); - }); - afterEach(function () { - document.body.removeChild(target); - target = null; - }); - - // The real constructors might not work everywhere we - // want to run these tests - const mouseevent = (typeArg, MouseEventInit) => { - const e = { type: typeArg }; - for (let key in MouseEventInit) { - e[key] = MouseEventInit[key]; - } - e.stopPropagation = sinon.spy(); - e.preventDefault = sinon.spy(); - return e; - }; - - describe('Decode Mouse Events', function () { - it('should decode mousedown events', function (done) { - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - expect(bmask).to.be.equal(0x01); - expect(down).to.be.equal(1); - done(); - }; - mouse._handleMouseDown(mouseevent('mousedown', { button: 0 })); - }); - it('should decode mouseup events', function (done) { - let calls = 0; - const mouse = new Mouse(target); - mouse.onmousebutton = (x, y, down, bmask) => { - expect(bmask).to.be.equal(0x01); - if (calls++ === 1) { - expect(down).to.not.be.equal(1); - done(); - } - }; - mouse._handleMouseDown(mouseevent('mousedown', { button: 0 })); - mouse._handleMouseUp(mouseevent('mouseup', { button: 0 })); - }); - it('should decode mousemove events', function (done) { - const mouse = new Mouse(target); - mouse.onmousemove = (x, y) => { - // Note that target relative coordinates are sent - expect(x).to.be.equal(40); - expect(y).to.be.equal(10); - done(); - }; - mouse._handleMouseMove(mouseevent('mousemove', - { clientX: 50, clientY: 20 })); - }); - }); -}); diff --git a/tests/test.rfb.js b/tests/test.rfb.js index cc450cd9..db9340ab 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -1579,12 +1579,10 @@ describe('Remote Frame Buffer Protocol Client', function () { expect(client._display.resize).to.have.been.calledWith(27, 32); }); - it('should grab the mouse and keyboard', function () { + it('should grab the keyboard', function () { sinon.spy(client._keyboard, 'grab'); - sinon.spy(client._mouse, 'grab'); sendServerInit({}, client); expect(client._keyboard.grab).to.have.been.calledOnce; - expect(client._mouse.grab).to.have.been.calledOnce; }); describe('Initial Update Request', function () { @@ -2739,6 +2737,11 @@ describe('Remote Frame Buffer Protocol Client', function () { client = makeRFB(); client._display.resize(100, 100); + // We need to disable this as focusing the canvas will + // cause the browser to scoll to it, messing up our + // client coordinate calculations + client.focusOnClick = false; + pointerEvent = sinon.spy(RFB.messages, 'pointerEvent'); keyEvent = sinon.spy(RFB.messages, 'keyEvent'); qemuKeyEvent = sinon.spy(RFB.messages, 'QEMUExtendedKeyEvent'); @@ -2769,136 +2772,279 @@ describe('Remote Frame Buffer Protocol Client', function () { } describe('Mouse Events', function () { + function sendMouseMoveEvent(x, y) { + let pos = elementToClient(x, y); + let ev; + + try { + ev = new MouseEvent('mousemove', + { 'screenX': pos.x + window.screenX, + 'screenY': pos.y + window.screenY, + 'clientX': pos.x, + 'clientY': pos.y }); + } catch (e) { + ev = document.createEvent('MouseEvent'); + ev.initMouseEvent('mousemove', + true, true, window, 0, + pos.x + window.screenX, + pos.y + window.screenY, + pos.x, pos.y, + false, false, false, false, + 0, null); + } + + client._canvas.dispatchEvent(ev); + } + + function sendMouseButtonEvent(x, y, down, button) { + let pos = elementToClient(x, y); + let ev; + + try { + ev = new MouseEvent(down ? 'mousedown' : 'mouseup', + { 'screenX': pos.x + window.screenX, + 'screenY': pos.y + window.screenY, + 'clientX': pos.x, + 'clientY': pos.y, + 'button': button, + 'buttons': 1 << button }); + } catch (e) { + ev = document.createEvent('MouseEvent'); + ev.initMouseEvent(down ? 'mousedown' : 'mouseup', + true, true, window, 0, + pos.x + window.screenX, + pos.y + window.screenY, + pos.x, pos.y, + false, false, false, false, + button, null); + } + + client._canvas.dispatchEvent(ev); + } + it('should not send button messages in view-only mode', function () { client._viewOnly = true; - client._handleMouseButton(0, 0, 1, 0x001); - expect(RFB.messages.pointerEvent).to.not.have.been.called; + sendMouseButtonEvent(10, 10, true, 0); + clock.tick(50); + expect(pointerEvent).to.not.have.been.called; }); it('should not send movement messages in view-only mode', function () { client._viewOnly = true; - client._handleMouseMove(0, 0); - expect(RFB.messages.pointerEvent).to.not.have.been.called; + sendMouseMoveEvent(10, 10); + clock.tick(50); + expect(pointerEvent).to.not.have.been.called; }); - it('should send a pointer event on mouse button presses', function () { - client._handleMouseButton(10, 12, 1, 0x001); - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; + it('should handle left mouse button', function () { + sendMouseButtonEvent(10, 10, true, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x1); + pointerEvent.resetHistory(); + + sendMouseButtonEvent(10, 10, false, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); }); - it('should send a mask of 1 on mousedown', function () { - client._handleMouseButton(11, 13, 1, 0x001); - expect(RFB.messages.pointerEvent).to.have.been.calledWith( - client._sock, 11, 13, 0x001); + it('should handle middle mouse button', function () { + sendMouseButtonEvent(10, 10, true, 1); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x2); + pointerEvent.resetHistory(); + + sendMouseButtonEvent(10, 10, false, 1); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); }); - it('should send a mask of 0 on mouseup', function () { - client._mouseButtonMask = 0x001; - client._handleMouseButton(105, 120, 0, 0x001); - expect(RFB.messages.pointerEvent).to.have.been.calledWith( - client._sock, 105, 120, 0x000); + it('should handle right mouse button', function () { + sendMouseButtonEvent(10, 10, true, 2); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x4); + pointerEvent.resetHistory(); + + sendMouseButtonEvent(10, 10, false, 2); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); }); - it('should send a mask of 0 on mousemove', function () { - client._handleMouseMove(100, 200); - expect(RFB.messages.pointerEvent).to.have.been.calledWith( - client._sock, 100, 200, 0x000); + it('should handle multiple mouse buttons', function () { + sendMouseButtonEvent(10, 10, true, 0); + sendMouseButtonEvent(10, 10, true, 2); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 0x1); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0x5); + + pointerEvent.resetHistory(); + + sendMouseButtonEvent(10, 10, false, 0); + sendMouseButtonEvent(10, 10, false, 2); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 0x4); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0x0); }); - it('should set the button mask so that future mouse movements use it', function () { - client._handleMouseButton(10, 12, 1, 0x010); - client._handleMouseMove(13, 9); - expect(RFB.messages.pointerEvent).to.have.been.calledTwice; - expect(RFB.messages.pointerEvent).to.have.been.calledWith( - client._sock, 13, 9, 0x010); + it('should handle mouse movement', function () { + sendMouseMoveEvent(50, 70); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 50, 70, 0x0); }); - it('should send a single pointer event on mouse movement', function () { - client._handleMouseMove(100, 200); - this.clock.tick(100); - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; + it('should handle click and drag', function () { + sendMouseButtonEvent(10, 10, true, 0); + sendMouseMoveEvent(50, 70); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 0x1); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 50, 70, 0x1); + + pointerEvent.resetHistory(); + + sendMouseButtonEvent(50, 70, false, 0); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 50, 70, 0x0); }); - it('should delay one move if two events are too close', function () { - client._handleMouseMove(18, 30); - client._handleMouseMove(20, 50); - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; - this.clock.tick(100); - expect(RFB.messages.pointerEvent).to.have.been.calledTwice; + describe('Event Aggregation', function () { + it('should send a single pointer event on mouse movement', function () { + sendMouseMoveEvent(50, 70); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 50, 70, 0x0); + }); + + it('should delay one move if two events are too close', function () { + sendMouseMoveEvent(18, 30); + sendMouseMoveEvent(20, 50); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 18, 30, 0x0); + pointerEvent.resetHistory(); + + clock.tick(100); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 20, 50, 0x0); + }); + + it('should only send first and last move of many close events', function () { + sendMouseMoveEvent(18, 30); + sendMouseMoveEvent(20, 50); + sendMouseMoveEvent(21, 55); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 18, 30, 0x0); + pointerEvent.resetHistory(); + + clock.tick(100); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 21, 55, 0x0); + }); + + // We selected the 17ms since that is ~60 FPS + it('should send move events every 17 ms', function () { + sendMouseMoveEvent(1, 10); // instant send + clock.tick(10); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 1, 10, 0x0); + pointerEvent.resetHistory(); + + sendMouseMoveEvent(2, 20); // delayed + clock.tick(10); // timeout send + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 2, 20, 0x0); + pointerEvent.resetHistory(); + + sendMouseMoveEvent(3, 30); // delayed + clock.tick(10); + sendMouseMoveEvent(4, 40); // delayed + clock.tick(10); // timeout send + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 4, 40, 0x0); + pointerEvent.resetHistory(); + + sendMouseMoveEvent(5, 50); // delayed + + expect(pointerEvent).to.not.have.been.called; + }); + + it('should send waiting move events before a button press', function () { + sendMouseMoveEvent(13, 9); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 13, 9, 0x0); + pointerEvent.resetHistory(); + + sendMouseMoveEvent(20, 70); + + expect(pointerEvent).to.not.have.been.called; + + sendMouseButtonEvent(20, 70, true, 0); + + expect(pointerEvent).to.have.been.calledTwice; + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 20, 70, 0x0); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 20, 70, 0x1); + }); + + it('should send move events with enough time apart normally', function () { + sendMouseMoveEvent(58, 60); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 58, 60, 0x0); + pointerEvent.resetHistory(); + + clock.tick(20); + + sendMouseMoveEvent(25, 60); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 25, 60, 0x0); + pointerEvent.resetHistory(); + }); + + it('should not send waiting move events if disconnected', function () { + sendMouseMoveEvent(88, 99); + + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 88, 99, 0x0); + pointerEvent.resetHistory(); + + sendMouseMoveEvent(66, 77); + client.disconnect(); + clock.tick(20); + + expect(pointerEvent).to.not.have.been.called; + }); }); - it('should only send first and last move of many close events', function () { - client._handleMouseMove(18, 40); - client._handleMouseMove(20, 50); - client._handleMouseMove(21, 55); - - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; - this.clock.tick(60); - - expect(RFB.messages.pointerEvent).to.have.been.calledTwice; - expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith( - client._sock, 18, 40, 0x000); - expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith( - client._sock, 21, 55, 0x000); + it.skip('should block click events', function () { + /* FIXME */ }); - // We selected the 17ms since that is ~60 FPS - it('should send move events every 17 ms', function () { - client._handleMouseMove(1, 10); // instant send - this.clock.tick(10); - client._handleMouseMove(2, 20); // delayed - this.clock.tick(10); // timeout send - client._handleMouseMove(3, 30); // delayed - this.clock.tick(10); - client._handleMouseMove(4, 40); // delayed - this.clock.tick(10); // timeout send - client._handleMouseMove(5, 50); // delayed - - expect(RFB.messages.pointerEvent).to.have.callCount(3); - expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith( - client._sock, 1, 10, 0x000); - expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith( - client._sock, 2, 20, 0x000); - expect(RFB.messages.pointerEvent.thirdCall).to.have.been.calledWith( - client._sock, 4, 40, 0x000); - }); - - it('should send waiting move events before a button press', function () { - client._handleMouseMove(13, 9); - client._handleMouseMove(20, 70); - client._handleMouseButton(10, 12, 1, 0x100); - expect(RFB.messages.pointerEvent).to.have.been.calledThrice; - expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith( - client._sock, 13, 9, 0x000); - expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith( - client._sock, 10, 12, 0x000); - expect(RFB.messages.pointerEvent.thirdCall).to.have.been.calledWith( - client._sock, 10, 12, 0x100); - }); - - it('should not delay events when button mask changes', function () { - client._handleMouseMove(13, 9); // instant - client._handleMouseMove(11, 10); // delayed - client._handleMouseButton(10, 12, 1, 0x010); // flush delayed - expect(RFB.messages.pointerEvent).to.have.been.calledThrice; - }); - - it('should send move events with enough time apart normally', function () { - client._handleMouseMove(58, 60); - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; - - this.clock.tick(20); - - client._handleMouseMove(25, 60); - expect(RFB.messages.pointerEvent).to.have.been.calledTwice; - }); - - it('should not send waiting move events if disconnected', function () { - client._handleMouseMove(88, 99); - client._handleMouseMove(66, 77); - client.disconnect(); - this.clock.tick(20); - expect(RFB.messages.pointerEvent).to.have.been.calledOnce; + it.skip('should block contextmenu events', function () { + /* FIXME */ }); }); From 32ed7c67249d876296fcb5d76052a84324b30e7b Mon Sep 17 00:00:00 2001 From: Samuel Mannehed Date: Thu, 11 Jun 2020 15:52:34 +0200 Subject: [PATCH 11/11] Fake cursor position when using touch With the new gestures we will simulate the cursor being in a different location than any of the touch points. This is a bit too complex for the Cursor class, so let's just explicitly tell it where we want the cursor rendered. --- core/rfb.js | 26 +++++++++++++++---------- core/util/cursor.js | 46 +++++++++++++-------------------------------- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index c56bdc98..4d1b6159 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1058,6 +1058,11 @@ export default class RFB extends EventTargetMixin { } } + _fakeMouseMove(ev, elementX, elementY) { + this._handleMouseMove(elementX, elementY); + this._cursor.move(ev.detail.clientX, ev.detail.clientY); + } + _handleTapEvent(ev, bmask) { let pos = clientToElement(ev.detail.clientX, ev.detail.clientY, this._canvas); @@ -1084,7 +1089,7 @@ export default class RFB extends EventTargetMixin { } this._gestureLastTapTime = Date.now(); - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(this._gestureFirstDoubleTapEv, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, true, bmask); this._handleMouseButton(pos.x, pos.y, false, bmask); } @@ -1107,23 +1112,23 @@ export default class RFB extends EventTargetMixin { this._handleTapEvent(ev, 0x2); break; case 'drag': - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, true, 0x1); break; case 'longpress': - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, true, 0x4); break; case 'twodrag': this._gestureLastMagnitudeX = ev.detail.magnitudeX; this._gestureLastMagnitudeY = ev.detail.magnitudeY; - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); break; case 'pinch': this._gestureLastMagnitudeX = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY); - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); break; } break; @@ -1136,13 +1141,13 @@ export default class RFB extends EventTargetMixin { break; case 'drag': case 'longpress': - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); break; case 'twodrag': // Always scroll in the same position. // We don't know if the mouse was moved so we need to move it // every update. - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); while ((ev.detail.magnitudeY - this._gestureLastMagnitudeY) > GESTURE_SCRLSENS) { this._handleMouseButton(pos.x, pos.y, true, 0x8); this._handleMouseButton(pos.x, pos.y, false, 0x8); @@ -1168,7 +1173,7 @@ export default class RFB extends EventTargetMixin { // Always scroll in the same position. // We don't know if the mouse was moved so we need to move it // every update. - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY); if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { this._handleKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); @@ -1184,6 +1189,7 @@ export default class RFB extends EventTargetMixin { } } this._handleKeyEvent(KeyTable.XK_Control_L, "ControlLeft", false); + break; } break; @@ -1196,11 +1202,11 @@ export default class RFB extends EventTargetMixin { case 'twodrag': break; case 'drag': - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, false, 0x1); break; case 'longpress': - this._handleMouseMove(pos.x, pos.y); + this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, false, 0x4); break; } diff --git a/core/util/cursor.js b/core/util/cursor.js index c7a084f0..3f0b01bd 100644 --- a/core/util/cursor.js +++ b/core/util/cursor.js @@ -30,9 +30,6 @@ export default class Cursor { 'mouseleave': this._handleMouseLeave.bind(this), 'mousemove': this._handleMouseMove.bind(this), 'mouseup': this._handleMouseUp.bind(this), - 'touchstart': this._handleTouchStart.bind(this), - 'touchmove': this._handleTouchMove.bind(this), - 'touchend': this._handleTouchEnd.bind(this), }; } @@ -54,11 +51,6 @@ export default class Cursor { this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options); this._target.addEventListener('mousemove', this._eventHandlers.mousemove, options); this._target.addEventListener('mouseup', this._eventHandlers.mouseup, options); - - // There is no "touchleave" so we monitor touchstart globally - window.addEventListener('touchstart', this._eventHandlers.touchstart, options); - this._target.addEventListener('touchmove', this._eventHandlers.touchmove, options); - this._target.addEventListener('touchend', this._eventHandlers.touchend, options); } this.clear(); @@ -76,10 +68,6 @@ export default class Cursor { this._target.removeEventListener('mousemove', this._eventHandlers.mousemove, options); this._target.removeEventListener('mouseup', this._eventHandlers.mouseup, options); - window.removeEventListener('touchstart', this._eventHandlers.touchstart, options); - this._target.removeEventListener('touchmove', this._eventHandlers.touchmove, options); - this._target.removeEventListener('touchend', this._eventHandlers.touchend, options); - document.body.removeChild(this._canvas); } @@ -131,6 +119,19 @@ export default class Cursor { this._hotSpot.y = 0; } + // Mouse events might be emulated, this allows + // moving the cursor in such cases + move(clientX, clientY) { + if (!useFallback) { + return; + } + this._position.x = clientX; + this._position.y = clientY; + this._updatePosition(); + let target = document.elementFromPoint(clientX, clientY); + this._updateVisibility(target); + } + _handleMouseOver(event) { // This event could be because we're entering the target, or // moving around amongst its sub elements. Let the move handler @@ -179,27 +180,6 @@ export default class Cursor { } } - _handleTouchStart(event) { - // Just as for mouseover, we let the move handler deal with it - this._handleTouchMove(event); - } - - _handleTouchMove(event) { - this._updateVisibility(event.target); - - this._position.x = event.changedTouches[0].clientX - this._hotSpot.x; - this._position.y = event.changedTouches[0].clientY - this._hotSpot.y; - - this._updatePosition(); - } - - _handleTouchEnd(event) { - // Same principle as for mouseup - let target = document.elementFromPoint(event.changedTouches[0].clientX, - event.changedTouches[0].clientY); - this._updateVisibility(target); - } - _showCursor() { if (this._canvas.style.visibility === 'hidden') { this._canvas.style.visibility = '';