From b475eed5fae5e7e1cdc80ae78604680b914d2a6e Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 28 Feb 2018 16:08:25 +0100 Subject: [PATCH 1/2] Separate out cursor handling Make cursor handling more generic in preparation for generic handling of corner cases. --- core/display.js | 53 --------------------------------- core/rfb.js | 16 +++++----- core/util/cursor.js | 71 ++++++++++++++++++++++++++++++++++++++++++++ docs/API-internal.md | 3 -- 4 files changed, 80 insertions(+), 63 deletions(-) create mode 100644 core/util/cursor.js diff --git a/core/display.js b/core/display.js index 99156157..44304ae5 100644 --- a/core/display.js +++ b/core/display.js @@ -498,18 +498,6 @@ Display.prototype = { this._damage(x, y, img.width, img.height); }, - changeCursor: function (pixels, mask, hotx, hoty, w, h) { - Display.changeCursor(this._target, pixels, mask, hotx, hoty, w, h); - }, - - defaultCursor: function () { - this._target.style.cursor = "default"; - }, - - disableLocalCursor: function () { - this._target.style.cursor = "none"; - }, - autoscale: function (containerWidth, containerHeight) { var vp = this._viewportLoc; var targetAspectRatio = containerWidth / containerHeight; @@ -655,44 +643,3 @@ Display.prototype = { } }, }; - -// Class Methods -Display.changeCursor = function (target, pixels, mask, hotx, hoty, w, h) { - if ((w === 0) || (h === 0)) { - target.style.cursor = 'none'; - return; - } - - var cur = [] - var y, x; - for (y = 0; y < h; y++) { - for (x = 0; x < w; x++) { - var idx = y * Math.ceil(w / 8) + Math.floor(x / 8); - var alpha = (mask[idx] << (x % 8)) & 0x80 ? 255 : 0; - idx = ((w * y) + x) * 4; - cur.push(pixels[idx + 2]); // red - cur.push(pixels[idx + 1]); // green - cur.push(pixels[idx]); // blue - cur.push(alpha); // alpha - } - } - - var canvas = document.createElement('canvas'); - var ctx = canvas.getContext('2d'); - - canvas.width = w; - canvas.height = h; - - var img; - if (SUPPORTS_IMAGEDATA_CONSTRUCTOR) { - img = new ImageData(new Uint8ClampedArray(cur), w, h); - } else { - img = ctx.createImageData(w, h); - img.data.set(new Uint8ClampedArray(cur)); - } - ctx.clearRect(0, 0, w, h); - ctx.putImageData(img, 0, 0); - - var url = canvas.toDataURL(); - target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; -}; diff --git a/core/rfb.js b/core/rfb.js index 81e1e6ad..741bf18d 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -17,6 +17,7 @@ import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Keyboard from "./input/keyboard.js"; import Mouse from "./input/mouse.js"; +import Cursor from "./util/cursor.js"; import Websock from "./websock.js"; import DES from "./des.js"; import KeyTable from "./input/keysym.js"; @@ -164,6 +165,8 @@ export default function RFB(target, url, options) { this._canvas.tabIndex = -1; this._screen.appendChild(this._canvas); + this._cursor = new Cursor(); + // populate encHandlers with bound versions this._encHandlers[encodings.encodingRaw] = RFB.encodingHandlers.RAW.bind(this); this._encHandlers[encodings.encodingCopyRect] = RFB.encodingHandlers.COPYRECT.bind(this); @@ -413,6 +416,8 @@ RFB.prototype = { // Make our elements part of the page this._target.appendChild(this._screen); + this._cursor.attach(this._canvas); + // Monitor size changes of the screen // FIXME: Use ResizeObserver, or hidden overflow window.addEventListener('resize', this._eventHandlers.windowResize); @@ -426,6 +431,7 @@ RFB.prototype = { _disconnect: function () { Log.Debug(">> RFB.disconnect"); + this._cursor.detach(); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); @@ -1241,10 +1247,6 @@ RFB.prototype = { this._timing.fbu_rt_start = (new Date()).getTime(); this._timing.pixels = 0; - // Cursor will be server side until the server decides to honor - // our request and send over the cursor image - this._display.disableLocalCursor(); - this._updateConnectionState('connected'); return true; }, @@ -2522,9 +2524,9 @@ RFB.encodingHandlers = { this._FBU.bytes = pixelslength + masklength; if (this._sock.rQwait("cursor encoding", this._FBU.bytes)) { return false; } - this._display.changeCursor(this._sock.rQshiftBytes(pixelslength), - this._sock.rQshiftBytes(masklength), - x, y, w, h); + this._cursor.change(this._sock.rQshiftBytes(pixelslength), + this._sock.rQshiftBytes(masklength), + x, y, w, h); this._FBU.bytes = 0; this._FBU.rects--; diff --git a/core/util/cursor.js b/core/util/cursor.js new file mode 100644 index 00000000..b7dcd5fa --- /dev/null +++ b/core/util/cursor.js @@ -0,0 +1,71 @@ +/* + * noVNC: HTML5 VNC client + * Copyright 2018 Pierre Ossman for noVNC + * Licensed under MPL 2.0 or any later version (see LICENSE.txt) + */ + +function Cursor(container) { + this._target = null; +} + +Cursor.prototype = { + attach: function (target) { + if (this._target) { + this.detach(); + } + + this._target = target; + + this.clear(); + }, + + detach: function () { + this._target = null; + }, + + change: function (pixels, mask, hotx, hoty, w, h) { + if ((w === 0) || (h === 0)) { + this.clear(); + return; + } + + let cur = [] + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + let idx = y * Math.ceil(w / 8) + Math.floor(x / 8); + let alpha = (mask[idx] << (x % 8)) & 0x80 ? 255 : 0; + idx = ((w * y) + x) * 4; + cur.push(pixels[idx + 2]); // red + cur.push(pixels[idx + 1]); // green + cur.push(pixels[idx]); // blue + cur.push(alpha); // alpha + } + } + + let canvas = document.createElement('canvas'); + let ctx = canvas.getContext('2d'); + + canvas.width = w; + canvas.height = h; + + let img; + try { + // IE doesn't support this + img = new ImageData(new Uint8ClampedArray(cur), w, h); + } catch (ex) { + img = ctx.createImageData(w, h); + img.data.set(new Uint8ClampedArray(cur)); + } + ctx.clearRect(0, 0, w, h); + ctx.putImageData(img, 0, 0); + + let url = this._canvas.toDataURL(); + this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; + }, + + clear: function () { + this._target.style.cursor = 'none'; + }, +}; + +export default Cursor; diff --git a/docs/API-internal.md b/docs/API-internal.md index 4943c1a1..0b29afb6 100644 --- a/docs/API-internal.md +++ b/docs/API-internal.md @@ -113,9 +113,6 @@ None | blitRgbImage | (x, y, width, height, arr, offset, from_queue) | Blit RGB encoded image to display | blitRgbxImage | (x, y, width, height, arr, offset, from_queue) | Blit RGBX encoded image to display | drawImage | (img, x, y) | Draw image and track damage -| changeCursor | (pixels, mask, hotx, hoty, w, h) | Change cursor appearance -| defaultCursor | () | Restore default cursor appearance -| disableLocalCursor | () | Disable local (client-side) cursor | autoscale | (containerWidth, containerHeight) | Scale the display ### 2.3.3 Callbacks From baa4f23ee520a41db0d793c0e2b39f8c9bccf517 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Wed, 28 Feb 2018 16:38:09 +0100 Subject: [PATCH 2/2] Provide fallback cursor method Some browsers don't support custom cursors, and there are cases where the browsers refuse to show the cursor. Handle both of these cases by letting the browser render the cursor via a floating canvas. This allows us to support a local cursor at all times. --- core/rfb.js | 4 +- core/util/cursor.js | 171 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 741bf18d..66db1a06 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -12,7 +12,6 @@ import * as Log from './util/logging.js'; import { decodeUTF8 } from './util/strings.js'; -import { supportsCursorURIs, isTouchDevice } from './util/browser.js'; import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Keyboard from "./input/keyboard.js"; @@ -1277,8 +1276,7 @@ RFB.prototype = { encs.push(encodings.pseudoEncodingFence); encs.push(encodings.pseudoEncodingContinuousUpdates); - if (supportsCursorURIs() && - !isTouchDevice && this._fb_depth == 24) { + if (this._fb_depth == 24) { encs.push(encodings.pseudoEncodingCursor); } diff --git a/core/util/cursor.js b/core/util/cursor.js index b7dcd5fa..da72723b 100644 --- a/core/util/cursor.js +++ b/core/util/cursor.js @@ -4,8 +4,36 @@ * Licensed under MPL 2.0 or any later version (see LICENSE.txt) */ +import { supportsCursorURIs, isTouchDevice } from './browser.js'; + +const useFallback = !supportsCursorURIs() || isTouchDevice; + function Cursor(container) { this._target = null; + + this._canvas = document.createElement('canvas'); + + if (useFallback) { + this._canvas.style.position = 'fixed'; + this._canvas.style.zIndex = '65535'; + this._canvas.style.pointerEvents = 'none'; + // Can't use "display" because of Firefox bug #1445997 + this._canvas.style.visibility = 'hidden'; + document.body.appendChild(this._canvas); + } + + this._position = { x: 0, y: 0 }; + this._hotSpot = { x: 0, y: 0 }; + + this._eventHandlers = { + 'mouseover': this._handleMouseOver.bind(this), + 'mouseleave': this._handleMouseLeave.bind(this), + 'mousemove': this._handleMouseMove.bind(this), + 'mouseup': this._handleMouseUp.bind(this), + 'touchstart': this._handleTouchStart.bind(this), + 'touchmove': this._handleTouchMove.bind(this), + 'touchend': this._handleTouchEnd.bind(this), + }; } Cursor.prototype = { @@ -16,10 +44,38 @@ Cursor.prototype = { this._target = target; + if (useFallback) { + // FIXME: These don't fire properly except for mouse + /// movement in IE. We want to also capture element + // movement, size changes, visibility, etc. + const options = { capture: true, passive: true }; + this._target.addEventListener('mouseover', this._eventHandlers.mouseover, options); + this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options); + this._target.addEventListener('mousemove', this._eventHandlers.mousemove, options); + this._target.addEventListener('mouseup', this._eventHandlers.mouseup, options); + + // There is no "touchleave" so we monitor touchstart globally + window.addEventListener('touchstart', this._eventHandlers.touchstart, options); + this._target.addEventListener('touchmove', this._eventHandlers.touchmove, options); + this._target.addEventListener('touchend', this._eventHandlers.touchend, options); + } + this.clear(); }, detach: function () { + if (useFallback) { + const options = { capture: true, passive: true }; + this._target.removeEventListener('mouseover', this._eventHandlers.mouseover, options); + this._target.removeEventListener('mouseleave', this._eventHandlers.mouseleave, options); + this._target.removeEventListener('mousemove', this._eventHandlers.mousemove, options); + this._target.removeEventListener('mouseup', this._eventHandlers.mouseup, options); + + window.removeEventListener('touchstart', this._eventHandlers.touchstart, options); + this._target.removeEventListener('touchmove', this._eventHandlers.touchmove, options); + this._target.removeEventListener('touchend', this._eventHandlers.touchend, options); + } + this._target = null; }, @@ -42,11 +98,15 @@ Cursor.prototype = { } } - let canvas = document.createElement('canvas'); - let ctx = canvas.getContext('2d'); + this._position.x = this._position.x + this._hotSpot.x - hotx; + this._position.y = this._position.y + this._hotSpot.y - hoty; + this._hotSpot.x = hotx; + this._hotSpot.y = hoty; - canvas.width = w; - canvas.height = h; + let ctx = this._canvas.getContext('2d'); + + this._canvas.width = w; + this._canvas.height = h; let img; try { @@ -59,12 +119,111 @@ Cursor.prototype = { ctx.clearRect(0, 0, w, h); ctx.putImageData(img, 0, 0); - let url = this._canvas.toDataURL(); - this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; + if (useFallback) { + this._updatePosition(); + } else { + let url = this._canvas.toDataURL(); + this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; + } }, clear: function () { this._target.style.cursor = 'none'; + this._canvas.width = 0; + this._canvas.height = 0; + this._position.x = this._position.x + this._hotSpot.x; + this._position.y = this._position.y + this._hotSpot.y; + this._hotSpot.x = 0; + this._hotSpot.y = 0; + }, + + _handleMouseOver: function (event) { + // This event could be because we're entering the target, or + // moving around amongst its sub elements. Let the move handler + // sort things out. + this._handleMouseMove(event); + }, + + _handleMouseLeave: function (event) { + this._hideCursor(); + }, + + _handleMouseMove: function (event) { + this._updateVisibility(event.target); + + this._position.x = event.clientX - this._hotSpot.x; + this._position.y = event.clientY - this._hotSpot.y; + + this._updatePosition(); + }, + + _handleMouseUp: function (event) { + // We might get this event because of a drag operation that + // moved outside of the target. Check what's under the cursor + // now and adjust visibility based on that. + let target = document.elementFromPoint(event.clientX, event.clientY); + this._updateVisibility(target); + }, + + _handleTouchStart: function (event) { + // Just as for mouseover, we let the move handler deal with it + this._handleTouchMove(event); + }, + + _handleTouchMove: function (event) { + this._updateVisibility(event.target); + + this._position.x = event.changedTouches[0].clientX - this._hotSpot.x; + this._position.y = event.changedTouches[0].clientY - this._hotSpot.y; + + this._updatePosition(); + }, + + _handleTouchEnd: function (event) { + // Same principle as for mouseup + let target = document.elementFromPoint(event.changedTouches[0].clientX, + event.changedTouches[0].clientY); + this._updateVisibility(target); + }, + + _showCursor: function () { + if (this._canvas.style.visibility === 'hidden') + this._canvas.style.visibility = ''; + }, + + _hideCursor: function () { + if (this._canvas.style.visibility !== 'hidden') + this._canvas.style.visibility = 'hidden'; + }, + + // Should we currently display the cursor? + // (i.e. are we over the target, or a child of the target without a + // different cursor set) + _shouldShowCursor: function (target) { + // Easy case + if (target === this._target) + return true; + // Other part of the DOM? + if (!this._target.contains(target)) + return false; + // Has the child its own cursor? + // FIXME: How can we tell that a sub element has an + // explicit "cursor: none;"? + if (window.getComputedStyle(target).cursor !== 'none') + return false; + return true; + }, + + _updateVisibility: function (target) { + if (this._shouldShowCursor(target)) + this._showCursor(); + else + this._hideCursor(); + }, + + _updatePosition: function () { + this._canvas.style.left = this._position.x + "px"; + this._canvas.style.top = this._position.y + "px"; }, };