From 7a4d1a82749060ba5f3167e387e1c39af5c8c9ff Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Thu, 9 Jan 2025 14:51:44 +0100 Subject: [PATCH 01/11] Move mouse event help functions to broader scope These functions can be used elsewhere in the tests. We want to use these in the dragging tests in the future instead of directly calling private methods. --- tests/test.rfb.js | 163 +++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 81 deletions(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 7e406321..c8d2693f 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -158,6 +158,50 @@ describe('Remote Frame Buffer protocol client', function () { return rfb; } + function elementToClient(x, y, client) { + let res = { x: 0, y: 0 }; + + let bounds = client._canvas.getBoundingClientRect(); + + /* + * If the canvas is on a fractional position we will calculate + * a fractional mouse position. But that gets truncated when we + * send the event, AND the same thing happens in RFB when it + * generates the PointerEvent message. To compensate for that + * fact we round the value upwards here. + */ + res.x = Math.ceil(bounds.left + x); + res.y = Math.ceil(bounds.top + y); + + return res; + } + + function sendMouseMoveEvent(x, y, client) { + let pos = elementToClient(x, y, client); + let ev; + + ev = new MouseEvent('mousemove', + { 'screenX': pos.x + window.screenX, + 'screenY': pos.y + window.screenY, + 'clientX': pos.x, + 'clientY': pos.y }); + client._canvas.dispatchEvent(ev); + } + + function sendMouseButtonEvent(x, y, down, button, client) { + let pos = elementToClient(x, y, client); + let ev; + + ev = new MouseEvent(down ? 'mousedown' : 'mouseup', + { 'screenX': pos.x + window.screenX, + 'screenY': pos.y + window.screenY, + 'clientX': pos.x, + 'clientY': pos.y, + 'button': button, + 'buttons': 1 << button }); + client._canvas.dispatchEvent(ev); + } + describe('Connecting/Disconnecting', function () { describe('#RFB (constructor)', function () { let open, attach; @@ -3584,107 +3628,64 @@ describe('Remote Frame Buffer protocol client', function () { qemuKeyEvent.restore(); }); - function elementToClient(x, y) { - let res = { x: 0, y: 0 }; - - let bounds = client._canvas.getBoundingClientRect(); - - /* - * If the canvas is on a fractional position we will calculate - * a fractional mouse position. But that gets truncated when we - * send the event, AND the same thing happens in RFB when it - * generates the PointerEvent message. To compensate for that - * fact we round the value upwards here. - */ - res.x = Math.ceil(bounds.left + x); - res.y = Math.ceil(bounds.top + y); - - return res; - } - describe('Mouse events', function () { - function sendMouseMoveEvent(x, y) { - let pos = elementToClient(x, y); - let ev; - - ev = new MouseEvent('mousemove', - { 'screenX': pos.x + window.screenX, - 'screenY': pos.y + window.screenY, - 'clientX': pos.x, - 'clientY': pos.y }); - client._canvas.dispatchEvent(ev); - } - - function sendMouseButtonEvent(x, y, down, button) { - let pos = elementToClient(x, y); - let ev; - - ev = new MouseEvent(down ? 'mousedown' : 'mouseup', - { 'screenX': pos.x + window.screenX, - 'screenY': pos.y + window.screenY, - 'clientX': pos.x, - 'clientY': pos.y, - 'button': button, - 'buttons': 1 << button }); - client._canvas.dispatchEvent(ev); - } it('should not send button messages in view-only mode', function () { client._viewOnly = true; - sendMouseButtonEvent(10, 10, true, 0); + sendMouseButtonEvent(10, 10, true, 0, client); clock.tick(50); expect(pointerEvent).to.not.have.been.called; }); it('should not send movement messages in view-only mode', function () { client._viewOnly = true; - sendMouseMoveEvent(10, 10); + sendMouseMoveEvent(10, 10, client); clock.tick(50); expect(pointerEvent).to.not.have.been.called; }); it('should handle left mouse button', function () { - sendMouseButtonEvent(10, 10, true, 0); + sendMouseButtonEvent(10, 10, true, 0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x1); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 0); + sendMouseButtonEvent(10, 10, false, 0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle middle mouse button', function () { - sendMouseButtonEvent(10, 10, true, 1); + sendMouseButtonEvent(10, 10, true, 1, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x2); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 1); + sendMouseButtonEvent(10, 10, false, 1, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle right mouse button', function () { - sendMouseButtonEvent(10, 10, true, 2); + sendMouseButtonEvent(10, 10, true, 2, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x4); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 2); + sendMouseButtonEvent(10, 10, false, 2, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle multiple mouse buttons', function () { - sendMouseButtonEvent(10, 10, true, 0); - sendMouseButtonEvent(10, 10, true, 2); + sendMouseButtonEvent(10, 10, true, 0, client); + sendMouseButtonEvent(10, 10, true, 2, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3694,8 +3695,8 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 0); - sendMouseButtonEvent(10, 10, false, 2); + sendMouseButtonEvent(10, 10, false, 0, client); + sendMouseButtonEvent(10, 10, false, 2, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3705,14 +3706,14 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should handle mouse movement', function () { - sendMouseMoveEvent(50, 70); + sendMouseMoveEvent(50, 70, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); }); it('should handle click and drag', function () { - sendMouseButtonEvent(10, 10, true, 0); - sendMouseMoveEvent(50, 70); + sendMouseButtonEvent(10, 10, true, 0, client); + sendMouseMoveEvent(50, 70, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3722,7 +3723,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - sendMouseButtonEvent(50, 70, false, 0); + sendMouseButtonEvent(50, 70, false, 0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); @@ -3730,15 +3731,15 @@ describe('Remote Frame Buffer protocol client', function () { describe('Event aggregation', function () { it('should send a single pointer event on mouse movement', function () { - sendMouseMoveEvent(50, 70); + sendMouseMoveEvent(50, 70, client); clock.tick(100); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); }); it('should delay one move if two events are too close', function () { - sendMouseMoveEvent(18, 30); - sendMouseMoveEvent(20, 50); + sendMouseMoveEvent(18, 30, client); + sendMouseMoveEvent(20, 50, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 18, 30, 0x0); @@ -3751,9 +3752,9 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should only send first and last move of many close events', function () { - sendMouseMoveEvent(18, 30); - sendMouseMoveEvent(20, 50); - sendMouseMoveEvent(21, 55); + sendMouseMoveEvent(18, 30, client); + sendMouseMoveEvent(20, 50, client); + sendMouseMoveEvent(21, 55, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 18, 30, 0x0); @@ -3767,46 +3768,46 @@ describe('Remote Frame Buffer protocol client', function () { // We selected the 17ms since that is ~60 FPS it('should send move events every 17 ms', function () { - sendMouseMoveEvent(1, 10); // instant send + sendMouseMoveEvent(1, 10, client); // instant send clock.tick(10); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 1, 10, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(2, 20); // delayed + sendMouseMoveEvent(2, 20, client); // delayed clock.tick(10); // timeout send expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 2, 20, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(3, 30); // delayed + sendMouseMoveEvent(3, 30, client); // delayed clock.tick(10); - sendMouseMoveEvent(4, 40); // delayed + sendMouseMoveEvent(4, 40, client); // delayed clock.tick(10); // timeout send expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 4, 40, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(5, 50); // delayed + sendMouseMoveEvent(5, 50, client); // delayed expect(pointerEvent).to.not.have.been.called; }); it('should send waiting move events before a button press', function () { - sendMouseMoveEvent(13, 9); + sendMouseMoveEvent(13, 9, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 13, 9, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(20, 70); + sendMouseMoveEvent(20, 70, client); expect(pointerEvent).to.not.have.been.called; - sendMouseButtonEvent(20, 70, true, 0); + sendMouseButtonEvent(20, 70, true, 0, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3816,7 +3817,7 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should send move events with enough time apart normally', function () { - sendMouseMoveEvent(58, 60); + sendMouseMoveEvent(58, 60, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 58, 60, 0x0); @@ -3824,7 +3825,7 @@ describe('Remote Frame Buffer protocol client', function () { clock.tick(20); - sendMouseMoveEvent(25, 60); + sendMouseMoveEvent(25, 60, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 25, 60, 0x0); @@ -3832,13 +3833,13 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should not send waiting move events if disconnected', function () { - sendMouseMoveEvent(88, 99); + sendMouseMoveEvent(88, 99, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 88, 99, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(66, 77); + sendMouseMoveEvent(66, 77, client); client.disconnect(); clock.tick(20); @@ -3857,7 +3858,7 @@ describe('Remote Frame Buffer protocol client', function () { describe('Wheel events', function () { function sendWheelEvent(x, y, dx, dy, mode=0) { - let pos = elementToClient(x, y); + let pos = elementToClient(x, y, client); let ev; ev = new WheelEvent('wheel', @@ -3990,7 +3991,7 @@ describe('Remote Frame Buffer protocol client', function () { describe('Gesture event handlers', function () { function gestureStart(gestureType, x, y, magnitudeX = 0, magnitudeY = 0) { - let pos = elementToClient(x, y); + let pos = elementToClient(x, y, client); let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; detail.magnitudeX = magnitudeX; @@ -4002,7 +4003,7 @@ describe('Remote Frame Buffer protocol client', function () { function gestureMove(gestureType, x, y, magnitudeX = 0, magnitudeY = 0) { - let pos = elementToClient(x, y); + let pos = elementToClient(x, y, client); let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; detail.magnitudeX = magnitudeX; @@ -4013,7 +4014,7 @@ describe('Remote Frame Buffer protocol client', function () { } function gestureEnd(gestureType, x, y) { - let pos = elementToClient(x, y); + let pos = elementToClient(x, y, client); let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; let ev = new CustomEvent('gestureend', { detail: detail }); client._canvas.dispatchEvent(ev); From de9d6888db04a6b43bc8a5d39f26c0dd51f71247 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Fri, 10 Jan 2025 14:22:14 +0100 Subject: [PATCH 02/11] Add unit test for wheel + buttons pressed --- tests/test.rfb.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index c8d2693f..66050bb0 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -3964,6 +3964,21 @@ describe('Remote Frame Buffer protocol client', function () { expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, 10, 10, 0); }); + + it('should handle wheel event with buttons pressed', function () { + sendMouseButtonEvent(10, 10, true, 0, client); + sendWheelEvent(10, 10, 0, 50); + + expect(pointerEvent).to.have.been.called.calledThrice; + + expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 10, 10, 0x1); + expect(pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 10, 10, 0x11); + expect(pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 10, 10, 0x1); + }); + }); describe('Keyboard events', function () { From db22ec6ee6ecc88e8d45f60a9022facd64784143 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Thu, 9 Jan 2025 14:59:19 +0100 Subject: [PATCH 03/11] Split button click with dragging test --- tests/test.rfb.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 66050bb0..861b808a 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -686,9 +686,9 @@ describe('Remote Frame Buffer protocol client', function () { client._handleMouseButton(13, 9, 0x001); client._handleMouseButton(13, 9, 0x000); expect(RFB.messages.pointerEvent).to.have.been.calledTwice; + }); - RFB.messages.pointerEvent.resetHistory(); - + it('should send button messages when release with small movement', function () { // Small movement client._handleMouseButton(13, 9, 0x001); client._handleMouseMove(15, 14); From c3934e0938ebe10e2749b8a51d70aa208dc73346 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Tue, 14 Jan 2025 12:14:34 +0100 Subject: [PATCH 04/11] Move mouse move flushing to separate function --- core/rfb.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 87fac3c2..3f946142 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1104,11 +1104,7 @@ export default class RFB extends EventTargetMixin { } // Flush waiting move event first - if (this._mouseMoveTimer !== null) { - clearTimeout(this._mouseMoveTimer); - this._mouseMoveTimer = null; - this._sendMouse(x, y, this._mouseButtonMask); - } + this._flushMouseMoveTimer(x, y); if (down) { this._mouseButtonMask |= bmask; @@ -1380,6 +1376,14 @@ export default class RFB extends EventTargetMixin { } } + _flushMouseMoveTimer(x, y) { + if (this._mouseMoveTimer !== null) { + clearTimeout(this._mouseMoveTimer); + this._mouseMoveTimer = null; + this._sendMouse(x, y, this._mouseButtonMask); + } + } + // Message handlers _negotiateProtocolVersion() { From dce8ab395b3efdc79ee5db12edfe390528bda6e5 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Thu, 9 Jan 2025 15:58:35 +0100 Subject: [PATCH 05/11] Dispatch mouse events in dragging unit tests This makes our tests reflect the real world better, as we now send real mouse events instead of calling private methods directly. --- tests/test.rfb.js | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 861b808a..061aff30 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -668,7 +668,10 @@ describe('Remote Frame Buffer protocol client', function () { describe('Dragging', function () { beforeEach(function () { + client = makeRFB(); client.dragViewport = true; + client._display.resize(100, 100); + sinon.spy(RFB.messages, "pointerEvent"); }); @@ -677,35 +680,39 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should not send button messages when initiating viewport dragging', function () { - client._handleMouseButton(13, 9, 0x001); + sendMouseButtonEvent(13, 9, true, 0, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; }); it('should send button messages when release without movement', function () { // Just up and down - client._handleMouseButton(13, 9, 0x001); - client._handleMouseButton(13, 9, 0x000); + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseButtonEvent(13, 9, false, 0, client); + expect(RFB.messages.pointerEvent).to.have.been.calledTwice; }); it('should send button messages when release with small movement', function () { // Small movement - client._handleMouseButton(13, 9, 0x001); - client._handleMouseMove(15, 14); - client._handleMouseButton(15, 14, 0x000); + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseMoveEvent(15, 14, client); + sendMouseButtonEvent(15, 14, false, 0, client); + expect(RFB.messages.pointerEvent).to.have.been.calledTwice; }); it('should not send button messages when in view only', function () { client._viewOnly = true; - client._handleMouseButton(13, 9, 0x001); - client._handleMouseButton(13, 9, 0x000); + + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseButtonEvent(13, 9, false, 0, client); + expect(RFB.messages.pointerEvent).to.not.have.been.called; }); it('should send button message directly when drag is disabled', function () { client.dragViewport = false; - client._handleMouseButton(13, 9, 0x001); + sendMouseButtonEvent(13, 9, true, 0, client); expect(RFB.messages.pointerEvent).to.have.been.calledOnce; }); @@ -713,16 +720,16 @@ describe('Remote Frame Buffer protocol client', function () { sinon.spy(client._display, "viewportChangePos"); // Too small movement + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseMoveEvent(18, 9, client); - client._handleMouseButton(13, 9, 0x001); - client._handleMouseMove(18, 9); expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.not.have.been.called; // Sufficient movement - client._handleMouseMove(43, 9); + sendMouseMoveEvent(43, 9, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.have.been.calledOnce; @@ -732,7 +739,7 @@ describe('Remote Frame Buffer protocol client', function () { // Now a small movement should move right away - client._handleMouseMove(43, 14); + sendMouseMoveEvent(43, 14, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.have.been.calledOnce; @@ -742,9 +749,9 @@ describe('Remote Frame Buffer protocol client', function () { it('should not send button messages when dragging ends', function () { // First the movement - client._handleMouseButton(13, 9, 0x001); - client._handleMouseMove(43, 9); - client._handleMouseButton(43, 9, 0x000); + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseMoveEvent(43, 9, client); + sendMouseButtonEvent(43, 9, false, 0, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; }); @@ -752,15 +759,15 @@ describe('Remote Frame Buffer protocol client', function () { it('should terminate viewport dragging on a button up event', function () { // First the dragging movement - client._handleMouseButton(13, 9, 0x001); - client._handleMouseMove(43, 9); - client._handleMouseButton(43, 9, 0x000); + sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseMoveEvent(43, 9, client); + sendMouseButtonEvent(43, 9, false, 0, client); // Another movement now should not move the viewport sinon.spy(client._display, "viewportChangePos"); - client._handleMouseMove(43, 59); + sendMouseMoveEvent(43, 59, client); expect(client._display.viewportChangePos).to.not.have.been.called; }); From 31d6a77af63feb211bcf5fc103b28eb755173fc4 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Fri, 10 Jan 2025 12:21:52 +0100 Subject: [PATCH 06/11] Check for correct button events in dragging tests Previously, these unit tests did not check which events were sent to the server, only how many events were sent. This commit adds checks to see that the expected button events are sent. --- tests/test.rfb.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 061aff30..7bd25e4e 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -690,6 +690,10 @@ describe('Remote Frame Buffer protocol client', function () { sendMouseButtonEvent(13, 9, false, 0, client); expect(RFB.messages.pointerEvent).to.have.been.calledTwice; + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 13, 9, 0x1); + expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 13, 9, 0x0); }); it('should send button messages when release with small movement', function () { @@ -699,6 +703,10 @@ describe('Remote Frame Buffer protocol client', function () { sendMouseButtonEvent(15, 14, false, 0, client); expect(RFB.messages.pointerEvent).to.have.been.calledTwice; + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 15, 14, 0x1); + expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 15, 14, 0x0); }); it('should not send button messages when in view only', function () { @@ -714,6 +722,8 @@ describe('Remote Frame Buffer protocol client', function () { client.dragViewport = false; sendMouseButtonEvent(13, 9, true, 0, client); expect(RFB.messages.pointerEvent).to.have.been.calledOnce; + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 13, 9, 0x1); }); it('should be initiate viewport dragging on sufficient movement', function () { From ea057d079329c9e22a3e940cfdb13eef132c9ad2 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Fri, 10 Jan 2025 13:14:49 +0100 Subject: [PATCH 07/11] Move gesture event help functions to broader scope This is needed if we want to test gestures with dragging. --- tests/test.rfb.js | 184 +++++++++++++++++++++++----------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 7bd25e4e..bdf86e60 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -202,6 +202,37 @@ describe('Remote Frame Buffer protocol client', function () { client._canvas.dispatchEvent(ev); } + function gestureStart(gestureType, x, y, client, + magnitudeX = 0, magnitudeY = 0) { + let pos = elementToClient(x, y, client); + let detail = { type: gestureType, clientX: pos.x, clientY: pos.y }; + + detail.magnitudeX = magnitudeX; + detail.magnitudeY = magnitudeY; + + let ev = new CustomEvent('gesturestart', { detail: detail }); + client._canvas.dispatchEvent(ev); + } + + function gestureMove(gestureType, x, y, client, + magnitudeX = 0, magnitudeY = 0) { + let pos = elementToClient(x, y, client); + let detail = { type: gestureType, clientX: pos.x, clientY: pos.y }; + + detail.magnitudeX = magnitudeX; + detail.magnitudeY = magnitudeY; + + let ev = new CustomEvent('gesturemove', { detail: detail }, client); + client._canvas.dispatchEvent(ev); + } + + function gestureEnd(gestureType, x, y, client) { + let pos = elementToClient(x, y, client); + let detail = { type: gestureType, clientX: pos.x, clientY: pos.y }; + let ev = new CustomEvent('gestureend', { detail: detail }); + client._canvas.dispatchEvent(ev); + } + describe('Connecting/Disconnecting', function () { describe('#RFB (constructor)', function () { let open, attach; @@ -4021,43 +4052,12 @@ describe('Remote Frame Buffer protocol client', function () { }); describe('Gesture event handlers', function () { - function gestureStart(gestureType, x, y, - magnitudeX = 0, magnitudeY = 0) { - let pos = elementToClient(x, y, client); - let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; - - detail.magnitudeX = magnitudeX; - detail.magnitudeY = magnitudeY; - - let ev = new CustomEvent('gesturestart', { detail: detail }); - client._canvas.dispatchEvent(ev); - } - - function gestureMove(gestureType, x, y, - magnitudeX = 0, magnitudeY = 0) { - let pos = elementToClient(x, y, client); - let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; - - detail.magnitudeX = magnitudeX; - detail.magnitudeY = magnitudeY; - - let ev = new CustomEvent('gesturemove', { detail: detail }); - client._canvas.dispatchEvent(ev); - } - - function gestureEnd(gestureType, x, y) { - let pos = elementToClient(x, y, client); - let detail = {type: gestureType, clientX: pos.x, clientY: pos.y}; - let ev = new CustomEvent('gestureend', { detail: detail }); - client._canvas.dispatchEvent(ev); - } - describe('Gesture onetap', function () { it('should handle onetap events', function () { let bmask = 0x1; - gestureStart('onetap', 20, 40); - gestureEnd('onetap', 20, 40); + gestureStart('onetap', 20, 40, client); + gestureEnd('onetap', 20, 40, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4071,8 +4071,8 @@ describe('Remote Frame Buffer protocol client', function () { it('should keep same position for multiple onetap events', function () { let bmask = 0x1; - gestureStart('onetap', 20, 40); - gestureEnd('onetap', 20, 40); + gestureStart('onetap', 20, 40, client); + gestureEnd('onetap', 20, 40, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4084,8 +4084,8 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureStart('onetap', 20, 50); - gestureEnd('onetap', 20, 50); + gestureStart('onetap', 20, 50, client); + gestureEnd('onetap', 20, 50, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4097,8 +4097,8 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureStart('onetap', 30, 50); - gestureEnd('onetap', 30, 50); + gestureStart('onetap', 30, 50, client); + gestureEnd('onetap', 30, 50, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4112,8 +4112,8 @@ describe('Remote Frame Buffer protocol client', function () { it('should not keep same position for onetap events when too far apart', function () { let bmask = 0x1; - gestureStart('onetap', 20, 40); - gestureEnd('onetap', 20, 40); + gestureStart('onetap', 20, 40, client); + gestureEnd('onetap', 20, 40, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4125,8 +4125,8 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureStart('onetap', 80, 95); - gestureEnd('onetap', 80, 95); + gestureStart('onetap', 80, 95, client); + gestureEnd('onetap', 80, 95, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4140,8 +4140,8 @@ describe('Remote Frame Buffer protocol client', function () { it('should not keep same position for onetap events when enough time inbetween', function () { let bmask = 0x1; - gestureStart('onetap', 10, 20); - gestureEnd('onetap', 10, 20); + gestureStart('onetap', 10, 20, client); + gestureEnd('onetap', 10, 20, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4154,8 +4154,8 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); this.clock.tick(1500); - gestureStart('onetap', 15, 20); - gestureEnd('onetap', 15, 20); + gestureStart('onetap', 15, 20, client); + gestureEnd('onetap', 15, 20, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4173,7 +4173,7 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture twotap events', function () { let bmask = 0x4; - gestureStart("twotap", 20, 40); + gestureStart("twotap", 20, 40, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4190,8 +4190,8 @@ describe('Remote Frame Buffer protocol client', function () { for (let offset = 0;offset < 30;offset += 10) { pointerEvent.resetHistory(); - gestureStart('twotap', 20, 40 + offset); - gestureEnd('twotap', 20, 40 + offset); + gestureStart('twotap', 20, 40 + offset, client); + gestureEnd('twotap', 20, 40 + offset, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4208,7 +4208,7 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture start for threetap events', function () { let bmask = 0x2; - gestureStart("threetap", 20, 40); + gestureStart("threetap", 20, 40, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4225,8 +4225,8 @@ describe('Remote Frame Buffer protocol client', function () { for (let offset = 0;offset < 30;offset += 10) { pointerEvent.resetHistory(); - gestureStart('threetap', 20, 40 + offset); - gestureEnd('threetap', 20, 40 + offset); + gestureStart('threetap', 20, 40 + offset, client); + gestureEnd('threetap', 20, 40 + offset, client); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4243,7 +4243,7 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture drag events', function () { let bmask = 0x1; - gestureStart('drag', 20, 40); + gestureStart('drag', 20, 40, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4253,7 +4253,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('drag', 30, 50); + gestureMove('drag', 30, 50, client); clock.tick(50); expect(pointerEvent).to.have.been.calledOnce; @@ -4262,7 +4262,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureEnd('drag', 30, 50); + gestureEnd('drag', 30, 50, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4276,7 +4276,7 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle long press events', function () { let bmask = 0x4; - gestureStart('longpress', 20, 40); + gestureStart('longpress', 20, 40, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4285,7 +4285,7 @@ describe('Remote Frame Buffer protocol client', function () { 20, 40, bmask); pointerEvent.resetHistory(); - gestureMove('longpress', 40, 60); + gestureMove('longpress', 40, 60, client); clock.tick(50); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, @@ -4293,7 +4293,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureEnd('longpress', 40, 60); + gestureEnd('longpress', 40, 60, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4307,14 +4307,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture twodrag up events', function () { let bmask = 0x10; // Button mask for scroll down - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 0, -60); + gestureMove('twodrag', 20, 40, client, 0, -60); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4328,14 +4328,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture twodrag down events', function () { let bmask = 0x8; // Button mask for scroll up - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 0, 60); + gestureMove('twodrag', 20, 40, client, 0, 60); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4349,14 +4349,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture twodrag right events', function () { let bmask = 0x20; // Button mask for scroll right - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 60, 0); + gestureMove('twodrag', 20, 40, client, 60, 0); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4370,14 +4370,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle gesture twodrag left events', function () { let bmask = 0x40; // Button mask for scroll left - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, -60, 0); + gestureMove('twodrag', 20, 40, client, -60, 0); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4392,14 +4392,14 @@ describe('Remote Frame Buffer protocol client', function () { let scrlUp = 0x8; // Button mask for scroll up let scrlRight = 0x20; // Button mask for scroll right - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 60, 60); + gestureMove('twodrag', 20, 40, client, 60, 60); expect(pointerEvent).to.have.been.callCount(5); expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, @@ -4417,14 +4417,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle multiple small gesture twodrag events', function () { let bmask = 0x8; // Button mask for scroll up - gestureStart('twodrag', 20, 40, 0, 0); + gestureStart('twodrag', 20, 40, client, 0, 0); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 0, 10); + gestureMove('twodrag', 20, 40, client, 0, 10); clock.tick(50); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, @@ -4432,7 +4432,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 0, 20); + gestureMove('twodrag', 20, 40, client, 0, 20); clock.tick(50); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, @@ -4440,7 +4440,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('twodrag', 20, 40, 0, 60); + gestureMove('twodrag', 20, 40, client, 0, 60); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4454,14 +4454,14 @@ describe('Remote Frame Buffer protocol client', function () { it('should handle large gesture twodrag events', function () { let bmask = 0x8; // Button mask for scroll up - gestureStart('twodrag', 30, 50, 0, 0); + gestureStart('twodrag', 30, 50, client, 0, 0); expect(pointerEvent). to.have.been.calledOnceWith(client._sock, 30, 50, 0x0); pointerEvent.resetHistory(); - gestureMove('twodrag', 30, 50, 0, 200); + gestureMove('twodrag', 30, 50, client, 0, 200); expect(pointerEvent).to.have.callCount(7); expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, @@ -4486,7 +4486,7 @@ describe('Remote Frame Buffer protocol client', function () { let keysym = KeyTable.XK_Control_L; let bmask = 0x10; // Button mask for scroll down - gestureStart('pinch', 20, 40, 90, 90); + gestureStart('pinch', 20, 40, client, 90, 90); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); @@ -4494,7 +4494,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 20, 40, 30, 30); + gestureMove('pinch', 20, 40, client, 30, 30); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4516,7 +4516,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); keyEvent.resetHistory(); - gestureEnd('pinch', 20, 40); + gestureEnd('pinch', 20, 40, client); expect(pointerEvent).to.not.have.been.called; expect(keyEvent).to.not.have.been.called; @@ -4526,7 +4526,7 @@ describe('Remote Frame Buffer protocol client', function () { let keysym = KeyTable.XK_Control_L; let bmask = 0x8; // Button mask for scroll up - gestureStart('pinch', 10, 20, 10, 20); + gestureStart('pinch', 10, 20, client, 10, 20); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 20, 0x0); @@ -4534,7 +4534,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 10, 20, 70, 80); + gestureMove('pinch', 10, 20, client, 70, 80); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4556,7 +4556,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); keyEvent.resetHistory(); - gestureEnd('pinch', 10, 20); + gestureEnd('pinch', 10, 20, client); expect(pointerEvent).to.not.have.been.called; expect(keyEvent).to.not.have.been.called; @@ -4566,7 +4566,7 @@ describe('Remote Frame Buffer protocol client', function () { let keysym = KeyTable.XK_Control_L; let bmask = 0x10; // Button mask for scroll down - gestureStart('pinch', 20, 40, 150, 150); + gestureStart('pinch', 20, 40, client, 150, 150); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); @@ -4574,7 +4574,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 20, 40, 30, 30); + gestureMove('pinch', 20, 40, client, 30, 30); expect(pointerEvent).to.have.been.callCount(5); expect(pointerEvent.getCall(0)).to.have.been.calledWith(client._sock, @@ -4600,7 +4600,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); keyEvent.resetHistory(); - gestureEnd('pinch', 20, 40); + gestureEnd('pinch', 20, 40, client); expect(pointerEvent).to.not.have.been.called; expect(keyEvent).to.not.have.been.called; @@ -4610,7 +4610,7 @@ describe('Remote Frame Buffer protocol client', function () { let keysym = KeyTable.XK_Control_L; let bmask = 0x8; // Button mask for scroll down - gestureStart('pinch', 20, 40, 0, 10); + gestureStart('pinch', 20, 40, client, 0, 10); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); @@ -4618,7 +4618,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 20, 40, 0, 30); + gestureMove('pinch', 20, 40, client, 0, 30); clock.tick(50); expect(pointerEvent).to.have.been.calledWith(client._sock, @@ -4626,7 +4626,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 20, 40, 0, 60); + gestureMove('pinch', 20, 40, client, 0, 60); clock.tick(50); expect(pointerEvent).to.have.been.calledWith(client._sock, @@ -4635,7 +4635,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); keyEvent.resetHistory(); - gestureMove('pinch', 20, 40, 0, 90); + gestureMove('pinch', 20, 40, client, 0, 90); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4657,7 +4657,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); keyEvent.resetHistory(); - gestureEnd('pinch', 20, 40); + gestureEnd('pinch', 20, 40, client); expect(keyEvent).to.not.have.been.called; }); @@ -4669,7 +4669,7 @@ describe('Remote Frame Buffer protocol client', function () { client._qemuExtKeyEventSupported = true; - gestureStart('pinch', 20, 40, 90, 90); + gestureStart('pinch', 20, 40, client, 90, 90); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 20, 40, 0x0); @@ -4677,7 +4677,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - gestureMove('pinch', 20, 40, 30, 30); + gestureMove('pinch', 20, 40, client, 30, 30); expect(pointerEvent).to.have.been.calledThrice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -4703,7 +4703,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); qemuKeyEvent.resetHistory(); - gestureEnd('pinch', 20, 40); + gestureEnd('pinch', 20, 40, client); expect(pointerEvent).to.not.have.been.called; expect(qemuKeyEvent).to.not.have.been.called; From f9eb476f6df2dc1bf05411d2efbc4ec49658939f Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Tue, 14 Jan 2025 09:39:03 +0100 Subject: [PATCH 08/11] Add tests for dragging with gestures There were no test for viewport dragging using gesture previously, so let's add some. Note that there currently are some viewport dragging behaviours that we don't want to have, so some tests have commented out what our desired behaviour should be. --- tests/test.rfb.js | 93 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/test.rfb.js b/tests/test.rfb.js index bdf86e60..b643e169 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -727,6 +727,20 @@ describe('Remote Frame Buffer protocol client', function () { 13, 9, 0x0); }); + it('should send button messages when tapping', function () { + // Just up and down + gestureStart('onetap', 13, 9, client); + gestureEnd('onetap', 13, 9, client); + + expect(RFB.messages.pointerEvent).to.have.been.calledThrice; + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 13, 9, 0x0); + expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 13, 9, 0x1); + expect(RFB.messages.pointerEvent.thirdCall).to.have.been.calledWith(client._sock, + 13, 9, 0x0); + }); + it('should send button messages when release with small movement', function () { // Small movement sendMouseButtonEvent(13, 9, true, 0, client); @@ -787,6 +801,85 @@ describe('Remote Frame Buffer protocol client', function () { expect(client._display.viewportChangePos).to.have.been.calledWith(0, -5); }); + it('should initiate viewport dragging on sufficient drag gesture movement', function () { + sinon.spy(client._display, "viewportChangePos"); + + // Sufficient movement + gestureStart('drag', 13, 9, client); + gestureMove('drag', 43, 9, client); + + // FIXME: We don't want to send a pointer event here + // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(client._display.viewportChangePos).to.have.been.calledOnce; + expect(client._display.viewportChangePos).to.have.been.calledWith(-30, 0); + + client._display.viewportChangePos.resetHistory(); + RFB.messages.pointerEvent.resetHistory(); + + // Now a small movement should move right away + + gestureMove('drag', 43, 14, client); + gestureEnd('drag', 43, 14, client); + + expect(RFB.messages.pointerEvent).to.not.have.been.called; + + // FIXME: We only want to move the viewport once + // expect(client._display.viewportChangePos).to.have.been.calledOnce; + expect(client._display.viewportChangePos).to.have.been.calledWith(0, -5); + }); + + it('should initiate viewport dragging on sufficient longpress gesture movement', function () { + sinon.spy(client._display, "viewportChangePos"); + + // A small movement below the threshold should not move. + gestureStart('longpress', 13, 9, client); + gestureMove('longpress', 14, 9, client); + + // FIXME: We don't want to send a pointer event here + // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(client._display.viewportChangePos).to.not.have.been.called; + + client._display.viewportChangePos.resetHistory(); + RFB.messages.pointerEvent.resetHistory(); + + gestureMove('longpress', 43, 9, client); + gestureEnd('longpress', 43, 9, client); + + expect(RFB.messages.pointerEvent).to.not.have.been.called; + // FIXME: We only want to move the viewport once + // expect(client._display.viewportChangePos).to.have.been.calledOnce; + expect(client._display.viewportChangePos).to.have.been.calledWith(-30, 0); + }); + + it('should send button messages on small longpress gesture movement', function () { + sinon.spy(client._display, "viewportChangePos"); + + // A small movement below the threshold should not move. + gestureStart('longpress', 13, 9, client); + gestureMove('longpress', 14, 10, client); + + // FIXME: We don't want to send a pointer event here + // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(client._display.viewportChangePos).to.not.have.been.called; + + client._display.viewportChangePos.resetHistory(); + RFB.messages.pointerEvent.resetHistory(); + + gestureEnd('longpress', 14, 9, client); + + // FIXME: We want the pointer event to come after the + // 'gestureEnd' call instead. + // expect(RFB.messages.pointerEvent).to.have.been.calledThrice; + // expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + // 14, 9, 0x0); + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 14, 9, 0x4); + expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 14, 9, 0x0); + + expect(client._display.viewportChangePos).to.not.have.been.called; + }); + it('should not send button messages when dragging ends', function () { // First the movement From b9230cf23eb0f57dec14b5d7f7fb6e753638b0f5 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Fri, 29 Nov 2024 14:47:19 +0100 Subject: [PATCH 09/11] Use MouseEvent.buttons for button state tracking Instead of keeping track of button states ourselves by looking at MouseEvent.button, we can use the MouseEvent.buttons which already contains the state of all buttons. --- core/rfb.js | 246 ++++++++++++++++++++++++++++++---------------- tests/test.rfb.js | 143 ++++++++++++++------------- 2 files changed, 234 insertions(+), 155 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 3f946142..4b105cb5 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1033,6 +1033,35 @@ export default class RFB extends EventTargetMixin { this.sendKey(keysym, code, down); } + static _convertButtonMask(buttons) { + /* The bits in MouseEvent.buttons property correspond + * to the following mouse buttons: + * 0: Left + * 1: Right + * 2: Middle + * 3: Back + * 4: Forward + * + * These bits needs to be converted to what they are defined as + * in the RFB protocol. + */ + + const buttonMaskMap = { + 0: 1 << 0, // Left + 1: 1 << 2, // Right + 2: 1 << 1, // Middle + 3: 1 << 7, // Back + }; + + let bmask = 0; + for (let i = 0; i < 4; i++) { + if (buttons & (1 << i)) { + bmask |= buttonMaskMap[i]; + } + } + return bmask; + } + _handleMouse(ev) { /* * We don't check connection status or viewOnly here as the @@ -1062,76 +1091,73 @@ export default class RFB extends EventTargetMixin { let pos = clientToElement(ev.clientX, ev.clientY, this._canvas); + let bmask = RFB._convertButtonMask(ev.buttons); + + let down = ev.type == 'mousedown'; switch (ev.type) { case 'mousedown': - setCapture(this._canvas); - this._handleMouseButton(pos.x, pos.y, - true, 1 << ev.button); - break; case 'mouseup': - this._handleMouseButton(pos.x, pos.y, - false, 1 << ev.button); + if (this.dragViewport) { + if (down && !this._viewportDragging) { + this._viewportDragging = true; + this._viewportDragPos = {'x': pos.x, 'y': pos.y}; + this._viewportHasMoved = false; + + // Skip sending mouse events, instead save the current + // mouse mask so we can send it later. + this._mouseButtonMask = bmask; + break; + } else { + this._viewportDragging = false; + + // If we actually performed a drag then we are done + // here and should not send any mouse events + if (this._viewportHasMoved) { + this._mouseButtonMask = bmask; + break; + } + // Otherwise we treat this as a mouse click event. + // Send the previously saved button mask, followed + // by the current button mask at the end of this + // function. + this._sendMouse(pos.x, pos.y, this._mouseButtonMask); + } + } + if (down) { + setCapture(this._canvas); + } + this._handleMouseButton(pos.x, pos.y, bmask); break; case 'mousemove': + if (this._viewportDragging) { + const deltaX = this._viewportDragPos.x - pos.x; + const deltaY = this._viewportDragPos.y - pos.y; + + if (this._viewportHasMoved || (Math.abs(deltaX) > dragThreshold || + Math.abs(deltaY) > dragThreshold)) { + this._viewportHasMoved = true; + + this._viewportDragPos = {'x': pos.x, 'y': pos.y}; + this._display.viewportChangePos(deltaX, deltaY); + } + + // Skip sending mouse events + break; + } this._handleMouseMove(pos.x, pos.y); break; } } - _handleMouseButton(x, y, down, bmask) { - if (this.dragViewport) { - if (down && !this._viewportDragging) { - this._viewportDragging = true; - this._viewportDragPos = {'x': x, 'y': y}; - this._viewportHasMoved = false; - - // Skip sending mouse events - return; - } else { - this._viewportDragging = false; - - // If we actually performed a drag then we are done - // here and should not send any mouse events - if (this._viewportHasMoved) { - return; - } - - // Otherwise we treat this as a mouse click event. - // Send the button down event here, as the button up - // event is sent at the end of this function. - this._sendMouse(x, y, bmask); - } - } - + _handleMouseButton(x, y, bmask) { // Flush waiting move event first this._flushMouseMoveTimer(x, y); - if (down) { - this._mouseButtonMask |= bmask; - } else { - this._mouseButtonMask &= ~bmask; - } - + this._mouseButtonMask = bmask; this._sendMouse(x, y, this._mouseButtonMask); } _handleMouseMove(x, y) { - if (this._viewportDragging) { - const deltaX = this._viewportDragPos.x - x; - const deltaY = this._viewportDragPos.y - y; - - if (this._viewportHasMoved || (Math.abs(deltaX) > dragThreshold || - Math.abs(deltaY) > dragThreshold)) { - this._viewportHasMoved = true; - - this._viewportDragPos = {'x': x, 'y': y}; - this._display.viewportChangePos(deltaX, deltaY); - } - - // Skip sending mouse events - return; - } - this._mousePos = { 'x': x, 'y': y }; // Limit many mouse move events to one every MOUSE_MOVE_DELAY ms @@ -1175,6 +1201,7 @@ export default class RFB extends EventTargetMixin { let pos = clientToElement(ev.clientX, ev.clientY, this._canvas); + let bmask = RFB._convertButtonMask(ev.buttons); let dX = ev.deltaX; let dY = ev.deltaY; @@ -1194,26 +1221,27 @@ export default class RFB extends EventTargetMixin { this._accumulatedWheelDeltaX += dX; this._accumulatedWheelDeltaY += dY; + // Generate a mouse wheel step event when the accumulated delta // for one of the axes is large enough. if (Math.abs(this._accumulatedWheelDeltaX) >= WHEEL_STEP) { if (this._accumulatedWheelDeltaX < 0) { - this._handleMouseButton(pos.x, pos.y, true, 1 << 5); - this._handleMouseButton(pos.x, pos.y, false, 1 << 5); + this._handleMouseButton(pos.x, pos.y, bmask | 1 << 5); + this._handleMouseButton(pos.x, pos.y, bmask); } else if (this._accumulatedWheelDeltaX > 0) { - this._handleMouseButton(pos.x, pos.y, true, 1 << 6); - this._handleMouseButton(pos.x, pos.y, false, 1 << 6); + this._handleMouseButton(pos.x, pos.y, bmask | 1 << 6); + this._handleMouseButton(pos.x, pos.y, bmask); } this._accumulatedWheelDeltaX = 0; } if (Math.abs(this._accumulatedWheelDeltaY) >= WHEEL_STEP) { if (this._accumulatedWheelDeltaY < 0) { - this._handleMouseButton(pos.x, pos.y, true, 1 << 3); - this._handleMouseButton(pos.x, pos.y, false, 1 << 3); + this._handleMouseButton(pos.x, pos.y, bmask | 1 << 3); + this._handleMouseButton(pos.x, pos.y, bmask); } else if (this._accumulatedWheelDeltaY > 0) { - this._handleMouseButton(pos.x, pos.y, true, 1 << 4); - this._handleMouseButton(pos.x, pos.y, false, 1 << 4); + this._handleMouseButton(pos.x, pos.y, bmask | 1 << 4); + this._handleMouseButton(pos.x, pos.y, bmask); } this._accumulatedWheelDeltaY = 0; @@ -1252,8 +1280,8 @@ export default class RFB extends EventTargetMixin { this._gestureLastTapTime = Date.now(); this._fakeMouseMove(this._gestureFirstDoubleTapEv, pos.x, pos.y); - this._handleMouseButton(pos.x, pos.y, true, bmask); - this._handleMouseButton(pos.x, pos.y, false, bmask); + this._handleMouseButton(pos.x, pos.y, bmask); + this._handleMouseButton(pos.x, pos.y, 0x0); } _handleGesture(ev) { @@ -1274,14 +1302,31 @@ export default class RFB extends EventTargetMixin { this._handleTapEvent(ev, 0x2); break; case 'drag': - this._fakeMouseMove(ev, pos.x, pos.y); - this._handleMouseButton(pos.x, pos.y, true, 0x1); + if (this.dragViewport) { + this._viewportHasMoved = false; + this._viewportDragging = true; + this._viewportDragPos = {'x': pos.x, 'y': pos.y}; + + this._fakeMouseMove(ev, pos.x, pos.y); + } else { + this._fakeMouseMove(ev, pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, 0x1); + } break; case 'longpress': - this._fakeMouseMove(ev, pos.x, pos.y); - this._handleMouseButton(pos.x, pos.y, true, 0x4); - break; + if (this.dragViewport) { + // If dragViewport is true, we need to wait to see + // if we have dragged outside the threshold before + // sending any events to the server. + this._viewportHasMoved = false; + this._viewportDragPos = {'x': pos.x, 'y': pos.y}; + this._fakeMouseMove(ev, pos.x, pos.y); + } else { + this._fakeMouseMove(ev, pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, 0x4); + } + break; case 'twodrag': this._gestureLastMagnitudeX = ev.detail.magnitudeX; this._gestureLastMagnitudeY = ev.detail.magnitudeY; @@ -1303,6 +1348,19 @@ export default class RFB extends EventTargetMixin { break; case 'drag': case 'longpress': + if (this.dragViewport) { + this._viewportDragging = true; + const deltaX = this._viewportDragPos.x - pos.x; + const deltaY = this._viewportDragPos.y - pos.y; + + if (this._viewportHasMoved || (Math.abs(deltaX) > dragThreshold || + Math.abs(deltaY) > dragThreshold)) { + this._viewportHasMoved = true; + + this._viewportDragPos = {'x': pos.x, 'y': pos.y}; + this._display.viewportChangePos(deltaX, deltaY); + } + } this._fakeMouseMove(ev, pos.x, pos.y); break; case 'twodrag': @@ -1311,23 +1369,23 @@ export default class RFB extends EventTargetMixin { // every update. this._fakeMouseMove(ev, pos.x, pos.y); while ((ev.detail.magnitudeY - this._gestureLastMagnitudeY) > GESTURE_SCRLSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x8); - this._handleMouseButton(pos.x, pos.y, false, 0x8); + this._handleMouseButton(pos.x, pos.y, 0x8); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeY += GESTURE_SCRLSENS; } while ((ev.detail.magnitudeY - this._gestureLastMagnitudeY) < -GESTURE_SCRLSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x10); - this._handleMouseButton(pos.x, pos.y, false, 0x10); + this._handleMouseButton(pos.x, pos.y, 0x10); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeY -= GESTURE_SCRLSENS; } while ((ev.detail.magnitudeX - this._gestureLastMagnitudeX) > GESTURE_SCRLSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x20); - this._handleMouseButton(pos.x, pos.y, false, 0x20); + this._handleMouseButton(pos.x, pos.y, 0x20); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeX += GESTURE_SCRLSENS; } while ((ev.detail.magnitudeX - this._gestureLastMagnitudeX) < -GESTURE_SCRLSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x40); - this._handleMouseButton(pos.x, pos.y, false, 0x40); + this._handleMouseButton(pos.x, pos.y, 0x40); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeX -= GESTURE_SCRLSENS; } break; @@ -1340,13 +1398,13 @@ export default class RFB extends EventTargetMixin { if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { this._handleKeyEvent(KeyTable.XK_Control_L, "ControlLeft", true); while ((magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x8); - this._handleMouseButton(pos.x, pos.y, false, 0x8); + this._handleMouseButton(pos.x, pos.y, 0x8); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeX += GESTURE_ZOOMSENS; } while ((magnitude - this._gestureLastMagnitudeX) < -GESTURE_ZOOMSENS) { - this._handleMouseButton(pos.x, pos.y, true, 0x10); - this._handleMouseButton(pos.x, pos.y, false, 0x10); + this._handleMouseButton(pos.x, pos.y, 0x10); + this._handleMouseButton(pos.x, pos.y, 0x0); this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS; } } @@ -1364,12 +1422,32 @@ export default class RFB extends EventTargetMixin { case 'twodrag': break; case 'drag': - this._fakeMouseMove(ev, pos.x, pos.y); - this._handleMouseButton(pos.x, pos.y, false, 0x1); + if (this.dragViewport) { + this._viewportDragging = false; + } else { + this._fakeMouseMove(ev, pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, 0x0); + } break; case 'longpress': - this._fakeMouseMove(ev, pos.x, pos.y); - this._handleMouseButton(pos.x, pos.y, false, 0x4); + if (this._viewportHasMoved) { + // We don't want to send any events if we have moved + // our viewport + break; + } + + if (this.dragViewport && !this._viewportHasMoved) { + this._fakeMouseMove(ev, pos.x, pos.y); + // If dragViewport is true, we need to wait to see + // if we have dragged outside the threshold before + // sending any events to the server. + this._handleMouseButton(pos.x, pos.y, 0x4); + this._handleMouseButton(pos.x, pos.y, 0x0); + this._viewportDragging = false; + } else { + this._fakeMouseMove(ev, pos.x, pos.y); + this._handleMouseButton(pos.x, pos.y, 0x0); + } break; } break; diff --git a/tests/test.rfb.js b/tests/test.rfb.js index b643e169..84298188 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -176,7 +176,7 @@ describe('Remote Frame Buffer protocol client', function () { return res; } - function sendMouseMoveEvent(x, y, client) { + function sendMouseMoveEvent(x, y, buttons, client) { let pos = elementToClient(x, y, client); let ev; @@ -184,11 +184,12 @@ describe('Remote Frame Buffer protocol client', function () { { 'screenX': pos.x + window.screenX, 'screenY': pos.y + window.screenY, 'clientX': pos.x, - 'clientY': pos.y }); + 'clientY': pos.y, + 'buttons': buttons }); client._canvas.dispatchEvent(ev); } - function sendMouseButtonEvent(x, y, down, button, client) { + function sendMouseButtonEvent(x, y, down, buttons, client) { let pos = elementToClient(x, y, client); let ev; @@ -197,8 +198,7 @@ describe('Remote Frame Buffer protocol client', function () { 'screenY': pos.y + window.screenY, 'clientX': pos.x, 'clientY': pos.y, - 'button': button, - 'buttons': 1 << button }); + 'buttons': buttons}); client._canvas.dispatchEvent(ev); } @@ -711,14 +711,14 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should not send button messages when initiating viewport dragging', function () { - sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; }); it('should send button messages when release without movement', function () { // Just up and down - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseButtonEvent(13, 9, false, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseButtonEvent(13, 9, false, 0x0, client); expect(RFB.messages.pointerEvent).to.have.been.calledTwice; expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -743,9 +743,9 @@ describe('Remote Frame Buffer protocol client', function () { it('should send button messages when release with small movement', function () { // Small movement - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseMoveEvent(15, 14, client); - sendMouseButtonEvent(15, 14, false, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseMoveEvent(15, 14, 0x1, client); + sendMouseButtonEvent(15, 14, false, 0x0, client); expect(RFB.messages.pointerEvent).to.have.been.calledTwice; expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -757,15 +757,15 @@ describe('Remote Frame Buffer protocol client', function () { it('should not send button messages when in view only', function () { client._viewOnly = true; - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseButtonEvent(13, 9, false, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseButtonEvent(13, 9, false, 0x0, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; }); it('should send button message directly when drag is disabled', function () { client.dragViewport = false; - sendMouseButtonEvent(13, 9, true, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); expect(RFB.messages.pointerEvent).to.have.been.calledOnce; expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, 13, 9, 0x1); @@ -775,8 +775,8 @@ describe('Remote Frame Buffer protocol client', function () { sinon.spy(client._display, "viewportChangePos"); // Too small movement - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseMoveEvent(18, 9, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseMoveEvent(18, 9, 0x1, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; @@ -784,7 +784,7 @@ describe('Remote Frame Buffer protocol client', function () { // Sufficient movement - sendMouseMoveEvent(43, 9, client); + sendMouseMoveEvent(43, 9, 0x1, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.have.been.calledOnce; @@ -794,7 +794,7 @@ describe('Remote Frame Buffer protocol client', function () { // Now a small movement should move right away - sendMouseMoveEvent(43, 14, client); + sendMouseMoveEvent(43, 14, 0x1, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.have.been.calledOnce; @@ -823,8 +823,7 @@ describe('Remote Frame Buffer protocol client', function () { expect(RFB.messages.pointerEvent).to.not.have.been.called; - // FIXME: We only want to move the viewport once - // expect(client._display.viewportChangePos).to.have.been.calledOnce; + expect(client._display.viewportChangePos).to.have.been.calledOnce; expect(client._display.viewportChangePos).to.have.been.calledWith(0, -5); }); @@ -867,14 +866,12 @@ describe('Remote Frame Buffer protocol client', function () { gestureEnd('longpress', 14, 9, client); - // FIXME: We want the pointer event to come after the - // 'gestureEnd' call instead. - // expect(RFB.messages.pointerEvent).to.have.been.calledThrice; - // expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, - // 14, 9, 0x0); + expect(RFB.messages.pointerEvent).to.have.been.calledThrice; expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, - 14, 9, 0x4); + 14, 9, 0x0); expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 14, 9, 0x4); + expect(RFB.messages.pointerEvent.thirdCall).to.have.been.calledWith(client._sock, 14, 9, 0x0); expect(client._display.viewportChangePos).to.not.have.been.called; @@ -883,9 +880,9 @@ describe('Remote Frame Buffer protocol client', function () { it('should not send button messages when dragging ends', function () { // First the movement - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseMoveEvent(43, 9, client); - sendMouseButtonEvent(43, 9, false, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseMoveEvent(43, 9, 0x1, client); + sendMouseButtonEvent(43, 9, false, 0x0, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; }); @@ -893,15 +890,15 @@ describe('Remote Frame Buffer protocol client', function () { it('should terminate viewport dragging on a button up event', function () { // First the dragging movement - sendMouseButtonEvent(13, 9, true, 0, client); - sendMouseMoveEvent(43, 9, client); - sendMouseButtonEvent(43, 9, false, 0, client); + sendMouseButtonEvent(13, 9, true, 0x1, client); + sendMouseMoveEvent(43, 9, 0x1, client); + sendMouseButtonEvent(43, 9, false, 0x0, client); // Another movement now should not move the viewport sinon.spy(client._display, "viewportChangePos"); - sendMouseMoveEvent(43, 59, client); + sendMouseMoveEvent(43, 59, 0x0, client); expect(client._display.viewportChangePos).to.not.have.been.called; }); @@ -3773,60 +3770,62 @@ describe('Remote Frame Buffer protocol client', function () { it('should not send button messages in view-only mode', function () { client._viewOnly = true; - sendMouseButtonEvent(10, 10, true, 0, client); + sendMouseButtonEvent(10, 10, true, 0x1, client); + clock.tick(50); expect(pointerEvent).to.not.have.been.called; }); it('should not send movement messages in view-only mode', function () { client._viewOnly = true; - sendMouseMoveEvent(10, 10, client); + sendMouseMoveEvent(10, 10, 0x0, client); + clock.tick(50); expect(pointerEvent).to.not.have.been.called; }); it('should handle left mouse button', function () { - sendMouseButtonEvent(10, 10, true, 0, client); + sendMouseButtonEvent(10, 10, true, 0x1, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x1); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 0, client); + sendMouseButtonEvent(10, 10, false, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle middle mouse button', function () { - sendMouseButtonEvent(10, 10, true, 1, client); + sendMouseButtonEvent(10, 10, true, 0x4, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x2); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 1, client); + sendMouseButtonEvent(10, 10, false, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle right mouse button', function () { - sendMouseButtonEvent(10, 10, true, 2, client); + sendMouseButtonEvent(10, 10, true, 0x2, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x4); pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 2, client); + sendMouseButtonEvent(10, 10, false, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 10, 10, 0x0); }); it('should handle multiple mouse buttons', function () { - sendMouseButtonEvent(10, 10, true, 0, client); - sendMouseButtonEvent(10, 10, true, 2, client); + sendMouseButtonEvent(10, 10, true, 0x1, client); + sendMouseButtonEvent(10, 10, true, 0x3, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3836,8 +3835,9 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - sendMouseButtonEvent(10, 10, false, 0, client); - sendMouseButtonEvent(10, 10, false, 2, client); + + sendMouseButtonEvent(10, 10, false, 0x2, client); + sendMouseButtonEvent(10, 10, false, 0x0, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3847,14 +3847,14 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should handle mouse movement', function () { - sendMouseMoveEvent(50, 70, client); + sendMouseMoveEvent(50, 70, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); }); it('should handle click and drag', function () { - sendMouseButtonEvent(10, 10, true, 0, client); - sendMouseMoveEvent(50, 70, client); + sendMouseButtonEvent(10, 10, true, 0x1, client); + sendMouseMoveEvent(50, 70, 0x1, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3864,7 +3864,7 @@ describe('Remote Frame Buffer protocol client', function () { pointerEvent.resetHistory(); - sendMouseButtonEvent(50, 70, false, 0, client); + sendMouseButtonEvent(50, 70, false, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); @@ -3872,15 +3872,15 @@ describe('Remote Frame Buffer protocol client', function () { describe('Event aggregation', function () { it('should send a single pointer event on mouse movement', function () { - sendMouseMoveEvent(50, 70, client); + sendMouseMoveEvent(50, 70, 0x0, client); clock.tick(100); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 50, 70, 0x0); }); it('should delay one move if two events are too close', function () { - sendMouseMoveEvent(18, 30, client); - sendMouseMoveEvent(20, 50, client); + sendMouseMoveEvent(18, 30, 0x0, client); + sendMouseMoveEvent(20, 50, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 18, 30, 0x0); @@ -3893,9 +3893,9 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should only send first and last move of many close events', function () { - sendMouseMoveEvent(18, 30, client); - sendMouseMoveEvent(20, 50, client); - sendMouseMoveEvent(21, 55, client); + sendMouseMoveEvent(18, 30, 0x0, client); + sendMouseMoveEvent(20, 50, 0x0, client); + sendMouseMoveEvent(21, 55, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 18, 30, 0x0); @@ -3909,46 +3909,46 @@ describe('Remote Frame Buffer protocol client', function () { // We selected the 17ms since that is ~60 FPS it('should send move events every 17 ms', function () { - sendMouseMoveEvent(1, 10, client); // instant send + sendMouseMoveEvent(1, 10, 0x0, client); // instant send clock.tick(10); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 1, 10, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(2, 20, client); // delayed + sendMouseMoveEvent(2, 20, 0x0, client); // delayed clock.tick(10); // timeout send expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 2, 20, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(3, 30, client); // delayed + sendMouseMoveEvent(3, 30, 0x0, client); // delayed clock.tick(10); - sendMouseMoveEvent(4, 40, client); // delayed + sendMouseMoveEvent(4, 40, 0x0, client); // delayed clock.tick(10); // timeout send expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 4, 40, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(5, 50, client); // delayed + sendMouseMoveEvent(5, 50, 0x0, client); // delayed expect(pointerEvent).to.not.have.been.called; }); it('should send waiting move events before a button press', function () { - sendMouseMoveEvent(13, 9, client); + sendMouseMoveEvent(13, 9, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 13, 9, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(20, 70, client); + sendMouseMoveEvent(20, 70, 0x0, client); expect(pointerEvent).to.not.have.been.called; - sendMouseButtonEvent(20, 70, true, 0, client); + sendMouseButtonEvent(20, 70, true, 0x1, client); expect(pointerEvent).to.have.been.calledTwice; expect(pointerEvent.firstCall).to.have.been.calledWith(client._sock, @@ -3958,7 +3958,7 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should send move events with enough time apart normally', function () { - sendMouseMoveEvent(58, 60, client); + sendMouseMoveEvent(58, 60, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 58, 60, 0x0); @@ -3966,7 +3966,7 @@ describe('Remote Frame Buffer protocol client', function () { clock.tick(20); - sendMouseMoveEvent(25, 60, client); + sendMouseMoveEvent(25, 60, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 25, 60, 0x0); @@ -3974,13 +3974,13 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should not send waiting move events if disconnected', function () { - sendMouseMoveEvent(88, 99, client); + sendMouseMoveEvent(88, 99, 0x0, client); expect(pointerEvent).to.have.been.calledOnceWith(client._sock, 88, 99, 0x0); pointerEvent.resetHistory(); - sendMouseMoveEvent(66, 77, client); + sendMouseMoveEvent(66, 77, 0x0, client); client.disconnect(); clock.tick(20); @@ -3998,7 +3998,7 @@ describe('Remote Frame Buffer protocol client', function () { }); describe('Wheel events', function () { - function sendWheelEvent(x, y, dx, dy, mode=0) { + function sendWheelEvent(x, y, dx, dy, mode=0, buttons=0) { let pos = elementToClient(x, y, client); let ev; @@ -4009,7 +4009,8 @@ describe('Remote Frame Buffer protocol client', function () { 'clientY': pos.y, 'deltaX': dx, 'deltaY': dy, - 'deltaMode': mode }); + 'deltaMode': mode, + 'buttons': buttons }); client._canvas.dispatchEvent(ev); } @@ -4107,8 +4108,8 @@ describe('Remote Frame Buffer protocol client', function () { }); it('should handle wheel event with buttons pressed', function () { - sendMouseButtonEvent(10, 10, true, 0, client); - sendWheelEvent(10, 10, 0, 50); + sendMouseButtonEvent(10, 10, true, 0x1, client); + sendWheelEvent(10, 10, 0, 50, 0, 0x1); expect(pointerEvent).to.have.been.called.calledThrice; From d1548c12ecb08a2aa5a871d8a621563f9688c1d1 Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Mon, 13 Jan 2025 15:43:47 +0100 Subject: [PATCH 10/11] Don't send mouse events when dragging viewport We don't want to send any mouse events to the server when dragging the viewport. Instead, we treat them as a client-only operation. --- core/rfb.js | 7 ++----- tests/test.rfb.js | 16 ++++++---------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 4b105cb5..5e08e32f 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1306,8 +1306,6 @@ export default class RFB extends EventTargetMixin { this._viewportHasMoved = false; this._viewportDragging = true; this._viewportDragPos = {'x': pos.x, 'y': pos.y}; - - this._fakeMouseMove(ev, pos.x, pos.y); } else { this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, 0x1); @@ -1320,8 +1318,6 @@ export default class RFB extends EventTargetMixin { // sending any events to the server. this._viewportHasMoved = false; this._viewportDragPos = {'x': pos.x, 'y': pos.y}; - - this._fakeMouseMove(ev, pos.x, pos.y); } else { this._fakeMouseMove(ev, pos.x, pos.y); this._handleMouseButton(pos.x, pos.y, 0x4); @@ -1360,8 +1356,9 @@ export default class RFB extends EventTargetMixin { this._viewportDragPos = {'x': pos.x, 'y': pos.y}; this._display.viewportChangePos(deltaX, deltaY); } + } else { + this._fakeMouseMove(ev, pos.x, pos.y); } - this._fakeMouseMove(ev, pos.x, pos.y); break; case 'twodrag': // Always scroll in the same position. diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 84298188..85c9eb66 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -808,8 +808,7 @@ describe('Remote Frame Buffer protocol client', function () { gestureStart('drag', 13, 9, client); gestureMove('drag', 43, 9, client); - // FIXME: We don't want to send a pointer event here - // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.have.been.calledOnce; expect(client._display.viewportChangePos).to.have.been.calledWith(-30, 0); @@ -834,8 +833,7 @@ describe('Remote Frame Buffer protocol client', function () { gestureStart('longpress', 13, 9, client); gestureMove('longpress', 14, 9, client); - // FIXME: We don't want to send a pointer event here - // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.not.have.been.called; client._display.viewportChangePos.resetHistory(); @@ -845,8 +843,7 @@ describe('Remote Frame Buffer protocol client', function () { gestureEnd('longpress', 43, 9, client); expect(RFB.messages.pointerEvent).to.not.have.been.called; - // FIXME: We only want to move the viewport once - // expect(client._display.viewportChangePos).to.have.been.calledOnce; + expect(client._display.viewportChangePos).to.have.been.calledOnce; expect(client._display.viewportChangePos).to.have.been.calledWith(-30, 0); }); @@ -857,8 +854,7 @@ describe('Remote Frame Buffer protocol client', function () { gestureStart('longpress', 13, 9, client); gestureMove('longpress', 14, 10, client); - // FIXME: We don't want to send a pointer event here - // expect(RFB.messages.pointerEvent).to.not.have.been.called; + expect(RFB.messages.pointerEvent).to.not.have.been.called; expect(client._display.viewportChangePos).to.not.have.been.called; client._display.viewportChangePos.resetHistory(); @@ -870,9 +866,9 @@ describe('Remote Frame Buffer protocol client', function () { expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, 14, 9, 0x0); expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, - 14, 9, 0x4); + 14, 9, 0x4); expect(RFB.messages.pointerEvent.thirdCall).to.have.been.calledWith(client._sock, - 14, 9, 0x0); + 14, 9, 0x0); expect(client._display.viewportChangePos).to.not.have.been.called; }); From 6383fa6384ed672c83f63693cdc3609851889a0a Mon Sep 17 00:00:00 2001 From: Adam Halim Date: Tue, 14 Jan 2025 12:32:51 +0100 Subject: [PATCH 11/11] Flush mouseMove when initiating viewport dragging We want to flush pending mouse moves before we initiate viewport dragging. Before this commit, there were scenarios where the _mouseButtonMask would track a released button as being down. --- core/rfb.js | 2 ++ tests/test.rfb.js | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/core/rfb.js b/core/rfb.js index 5e08e32f..89e9197d 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1103,6 +1103,8 @@ export default class RFB extends EventTargetMixin { this._viewportDragPos = {'x': pos.x, 'y': pos.y}; this._viewportHasMoved = false; + this._flushMouseMoveTimer(pos.x, pos.y); + // Skip sending mouse events, instead save the current // mouse mask so we can send it later. this._mouseButtonMask = bmask; diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 85c9eb66..62f2a649 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -898,6 +898,24 @@ describe('Remote Frame Buffer protocol client', function () { expect(client._display.viewportChangePos).to.not.have.been.called; }); + + it('should flush move events when initiating viewport drag', function () { + sendMouseMoveEvent(13, 9, 0x0, client); + sendMouseMoveEvent(14, 9, 0x0, client); + sendMouseButtonEvent(14, 9, true, 0x1, client); + + expect(RFB.messages.pointerEvent).to.have.been.calledTwice; + expect(RFB.messages.pointerEvent.firstCall).to.have.been.calledWith(client._sock, + 13, 9, 0x0); + expect(RFB.messages.pointerEvent.secondCall).to.have.been.calledWith(client._sock, + 14, 9, 0x0); + + RFB.messages.pointerEvent.resetHistory(); + + clock.tick(100); + + expect(RFB.messages.pointerEvent).to.not.have.been.called;; + }); }); });