From 385a1f99b40a44e178d8ec167e50a8a7c92da2f0 Mon Sep 17 00:00:00 2001 From: Matt McClaskey Date: Fri, 25 Mar 2022 15:03:19 -0400 Subject: [PATCH] Feature/kasm 2335 ime support 2 (#27) IME support, refactored keyboard input --- app/styles/base.css | 17 ++-- app/ui.js | 167 ++++++++++++++----------------- core/input/imekeys.js | 32 ++++++ core/input/keyboard.js | 219 +++++++++++++++++++++++++++++++++++++++-- core/rfb.js | 12 ++- core/util/browser.js | 4 + vnc.html | 8 +- 7 files changed, 344 insertions(+), 115 deletions(-) create mode 100644 core/input/imekeys.js diff --git a/app/styles/base.css b/app/styles/base.css index 8f272347..e83e08cd 100644 --- a/app/styles/base.css +++ b/app/styles/base.css @@ -936,15 +936,15 @@ select:active { } #noVNC_keyboardinput { - width: 1px; - height: 1px; - background-color: #fff; - color: #fff; + width: 0px; + height: 0px; + background-color: #fff0; + color: rgba(5, 5, 5, 0); border: 0; position: absolute; - left: -40px; + left: 35%; + top: 40%; z-index: -1; - ime-mode: disabled; } /*Default noVNC logo.*/ @@ -1021,6 +1021,11 @@ body { user-select: none; } +#noVNC_keyboard_control .noVNC_selected { + background-color:rgb(15, 36, 153); + border: 6px rgb(15, 36, 153) solid; +} + .keyboard-controls .button.ctrl { background-image: url("../images/ctrl.svg"); background-size: contain; diff --git a/app/ui.js b/app/ui.js index 3d5bb0f0..41e31c64 100644 --- a/app/ui.js +++ b/app/ui.js @@ -65,8 +65,6 @@ const UI = { controlbarMouseDownClientY: 0, controlbarMouseDownOffsetY: 0, - lastKeyboardinput: null, - defaultKeyboardinputLen: 100, needToCheckClipboardChange: false, inhibitReconnect: true, @@ -135,10 +133,6 @@ const UI = { UI.addSettingsHandlers(); document.getElementById("noVNC_status") .addEventListener('click', UI.hideStatus); - - // Bootstrap fallback input handler - UI.keyboardinputReset(); - UI.openControlbar(); UI.updateVisualState('init'); @@ -245,6 +239,9 @@ const UI = { UI.initSetting('prefer_local_cursor', true); UI.initSetting('toggle_control_panel', false); UI.initSetting('enable_perf_stats', false); + UI.initSetting('virtual_keyboard_visible', false); + UI.initSetting('enable_ime', false) + UI.toggleKeyboardControls(); if (WebUtil.isInsideKasmVDI()) { UI.initSetting('clipboard_up', false); @@ -371,12 +368,6 @@ const UI = { addTouchSpecificHandlers() { document.getElementById("noVNC_keyboard_button") .addEventListener('click', UI.toggleVirtualKeyboard); - - UI.touchKeyboard = new Keyboard(document.getElementById('noVNC_keyboardinput')); - UI.touchKeyboard.onkeyevent = UI.keyEvent; - UI.touchKeyboard.grab(); - document.getElementById("noVNC_keyboardinput") - .addEventListener('input', UI.keyInput); document.getElementById("noVNC_keyboardinput") .addEventListener('focus', UI.onfocusVirtualKeyboard); document.getElementById("noVNC_keyboardinput") @@ -536,6 +527,10 @@ const UI = { UI.addSettingChangeHandler('clipboard_seamless'); UI.addSettingChangeHandler('clipboard_up'); UI.addSettingChangeHandler('clipboard_down'); + UI.addSettingChangeHandler('virtual_keyboard_visible'); + UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls); + UI.addSettingChangeHandler('enable_ime'); + UI.addSettingChangeHandler('enable_ime', UI.toggleIMEMode); }, addFullscreenHandlers() { @@ -1297,7 +1292,9 @@ const UI = { } url += '/' + path; - UI.rfb = new RFB(document.getElementById('noVNC_container'), url, + UI.rfb = new RFB(document.getElementById('noVNC_container'), + document.getElementById('noVNC_keyboardinput'), + url, { shared: UI.getSetting('shared'), repeaterID: UI.getSetting('repeaterID'), credentials: { password: password } }); @@ -1335,6 +1332,7 @@ const UI = { UI.rfb.clipboardUp = UI.getSetting('clipboard_up'); UI.rfb.clipboardDown = UI.getSetting('clipboard_down'); UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless'); + UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime'); UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless; //Only explicitly request permission to clipboard on browsers that support binary clipboard access @@ -1544,6 +1542,30 @@ const UI = { UI.forceSetting('video_quality', parseInt(event.data.value), false); UI.updateQuality(); break; + case 'show_keyboard_controls': + if (!UI.getSetting('virtual_keyboard_visible')) { + UI.forceSetting('virtual_keyboard_visible', true, false); + UI.showKeyboardControls(); + } + break; + case 'hide_keyboard_controls': + if (UI.getSetting('virtual_keyboard_visible')) { + UI.forceSetting('virtual_keyboard_visible', true, false); + UI.hideKeyboardControls(); + } + break; + case 'enable_ime_mode': + if (!UI.getSetting('enable_ime')) { + UI.forceSetting('enable_ime', true, false); + UI.toggleIMEMode(); + } + break; + case 'disable_ime_mode': + if (UI.getSetting('enable_ime')) { + UI.forceSetting('enable_ime', false, false); + UI.toggleIMEMode(); + } + break; } } }, @@ -1881,18 +1903,41 @@ const UI = { UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts'); }, + toggleKeyboardControls() { + if (UI.getSetting('virtual_keyboard_visible')) { + UI.showKeyboardControls(); + } else { + UI.hideKeyboardControls(); + } + }, + + toggleIMEMode() { + if (UI.rfb) { + if (UI.getSetting('enable_ime')) { + UI.rfb.keyboard.enableIME = true; + } else { + UI.rfb.keyboard.enableIME = false; + } + } + }, + showKeyboardControls() { - document.querySelector(".keyboard-controls").classList.add("is-visible"); + document.getElementById('noVNC_keyboard_control').classList.add("is-visible"); }, hideKeyboardControls() { - document.querySelector(".keyboard-controls").classList.remove("is-visible"); + document.getElementById('noVNC_keyboard_control').classList.remove("is-visible"); }, showVirtualKeyboard() { const input = document.getElementById('noVNC_keyboardinput'); - if (document.activeElement == input) return; + if (document.activeElement == input || !UI.rfb) return; + + if (UI.getSetting('virtual_keyboard_visible')) { + document.getElementById('noVNC_keyboard_control_handle') + .classList.add("noVNC_selected"); + } input.focus(); @@ -1916,7 +1961,12 @@ const UI = { hideVirtualKeyboard() { const input = document.getElementById('noVNC_keyboardinput'); - if (document.activeElement != input) return; + if (document.activeElement != input || !UI.rfb) return; + + if (UI.getSetting('virtual_keyboard_visible')) { + document.getElementById('noVNC_keyboard_control_handle') + .classList.remove("noVNC_selected"); + } input.blur(); }, @@ -1941,6 +1991,12 @@ const UI = { onblurVirtualKeyboard(event) { document.getElementById('noVNC_keyboard_button') .classList.remove("noVNC_selected"); + + if (UI.getSetting('virtual_keyboard_visible')) { + document.getElementById('noVNC_keyboard_control_handle') + .classList.remove("noVNC_selected"); + } + if (UI.rfb) { UI.rfb.focusOnClick = true; } @@ -1974,83 +2030,6 @@ const UI = { event.preventDefault(); }, - keyboardinputReset() { - const kbi = document.getElementById('noVNC_keyboardinput'); - kbi.value = new Array(UI.defaultKeyboardinputLen).join("_"); - UI.lastKeyboardinput = kbi.value; - }, - - keyEvent(keysym, code, down) { - if (!UI.rfb) return; - - UI.rfb.sendKey(keysym, code, down); - }, - - // When normal keyboard events are left uncought, use the input events from - // the keyboardinput element instead and generate the corresponding key events. - // This code is required since some browsers on Android are inconsistent in - // sending keyCodes in the normal keyboard events when using on screen keyboards. - keyInput(event) { - - if (!UI.rfb) return; - - const newValue = event.target.value; - - if (!UI.lastKeyboardinput) { - UI.keyboardinputReset(); - } - const oldValue = UI.lastKeyboardinput; - - let newLen; - try { - // Try to check caret position since whitespace at the end - // will not be considered by value.length in some browsers - newLen = Math.max(event.target.selectionStart, newValue.length); - } catch (err) { - // selectionStart is undefined in Google Chrome - newLen = newValue.length; - } - const oldLen = oldValue.length; - - let inputs = newLen - oldLen; - let backspaces = inputs < 0 ? -inputs : 0; - - // Compare the old string with the new to account for - // text-corrections or other input that modify existing text - for (let i = 0; i < Math.min(oldLen, newLen); i++) { - if (newValue.charAt(i) != oldValue.charAt(i)) { - inputs = newLen - i; - backspaces = oldLen - i; - break; - } - } - - // Send the key events - for (let i = 0; i < backspaces; i++) { - UI.rfb.sendKey(KeyTable.XK_BackSpace, "Backspace"); - } - for (let i = newLen - inputs; i < newLen; i++) { - UI.rfb.sendKey(keysyms.lookup(newValue.charCodeAt(i))); - } - - // Control the text content length in the keyboardinput element - if (newLen > 2 * UI.defaultKeyboardinputLen) { - UI.keyboardinputReset(); - } else if (newLen < 1) { - // There always have to be some text in the keyboardinput - // element with which backspace can interact. - UI.keyboardinputReset(); - // This sometimes causes the keyboard to disappear for a second - // but it is required for the android keyboard to recognize that - // text has been added to the field - event.target.blur(); - // This has to be ran outside of the input handler in order to work - setTimeout(event.target.focus.bind(event.target), 0); - } else { - UI.lastKeyboardinput = newValue; - } - }, - /* ------^------- * /KEYBOARD * ============== diff --git a/core/input/imekeys.js b/core/input/imekeys.js new file mode 100644 index 00000000..219e238a --- /dev/null +++ b/core/input/imekeys.js @@ -0,0 +1,32 @@ +/* + * KasmVNC: HTML5 VNC client + * Copyright (C) 2022 Kasm Technologies Inc + * Licensed under MPL 2.0 or any later version (see LICENSE.txt) + */ + +/* + * Keys that could be interaction with IME input + */ + +export default { + 0x30: 'Digit0', + 0x31: 'Digit1', + 0x32: 'Digit2', + 0x33: 'Digit3', + 0x34: 'Digit4', + 0x35: 'Digit5', + 0x36: 'Digit6', + 0x37: 'Digit7', + 0x38: 'Digit8', + 0x39: 'Digit9', + 0x60: 'Numpad0', + 0x61: 'Numpad1', + 0x62: 'Numpad2', + 0x63: 'Numpad3', + 0x64: 'Numpad4', + 0x65: 'Numpad5', + 0x66: 'Numpad6', + 0x67: 'Numpad7', + 0x68: 'Numpad8', + 0x69: 'Numpad9' +}; \ No newline at end of file diff --git a/core/input/keyboard.js b/core/input/keyboard.js index 82b5f5c1..4cd0b28c 100644 --- a/core/input/keyboard.js +++ b/core/input/keyboard.js @@ -8,16 +8,20 @@ import * as Log from '../util/logging.js'; import { stopEvent } from '../util/events.js'; import * as KeyboardUtil from "./util.js"; import KeyTable from "./keysym.js"; +import keysyms from "./keysymdef.js"; +import imekeys from "./imekeys.js"; import * as browser from "../util/browser.js"; import UI from '../../app/ui.js'; +import { isChromiumBased } from '../util/browser.js'; // // Keyboard event handler // export default class Keyboard { - constructor(target) { - this._target = target || null; + constructor(screenInput, touchInput) { + this._screenInput = screenInput; + this._touchInput = touchInput; this._keyDownList = {}; // List of depressed keys // (even if they are happy) @@ -28,11 +32,28 @@ export default class Keyboard { 'keyup': this._handleKeyUp.bind(this), 'keydown': this._handleKeyDown.bind(this), 'blur': this._allKeysUp.bind(this), + 'compositionstart' : this._handleCompositionStart.bind(this), + 'compositionend' : this._handleCompositionEnd.bind(this), + 'input' : this._handleInput.bind(this) }; // ===== EVENT HANDLERS ===== - this.onkeyevent = () => {}; // Handler for key press/release + + this._enableIME = false; + this._imeHold = false; + this._imeInProgress = false; + this._lastKeyboardInput = null; + this._defaultKeyboardInputLen = 100; + this._keyboardInputReset(); + } + + // ===== PUBLIC METHODS ===== + + get enableIME() { return this._enableIME; } + set enableIME(val) { + this._enableIME = val; + this.focus(); } // ===== PRIVATE METHODS ===== @@ -95,10 +116,135 @@ export default class Keyboard { return 'Unidentified'; } + _handleCompositionStart(e) { + Log.Debug("composition started"); + if (this._enableIME) { + this._imeHold = true; + this._imeInProgress = true; + } + } + + _handleCompositionEnd(e) { + Log.Debug("Composition ended"); + if (this._enableIME) { this._imeInProgress = false; } + if (isChromiumBased()) { + this._imeHold = false; + } + } + + _handleInput(e) { + //input event occurs only when keyup keydown events don't prevent default + //IME events will make this happen, for example + //IME changes can back out old characters and replace, thus send differential if IME + //otherwise send new characters + if (this._enableIME && this._imeHold) { + Log.Debug("IME input change, sending differential"); + if (!this._imeInProgress) { + this._imeHold = false; //Firefox fires compisitionend before last input change + } + + const oldValue = this._lastKeyboardInput; + const newValue = e.target.value; + let diff_start = 0; + + //find position where difference starts + for (let i = 0; i < Math.min(oldValue.length, newValue.length); i++) { + if (newValue.charAt(i) != oldValue.charAt(i)) { + break; + } + diff_start++; + } + + //send backspaces if needed + for (let bs = oldValue.length - diff_start; bs > 0; bs--) { + this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", true); + this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", false); + } + + //send new keys + for (let i = diff_start; i < newValue.length; i++) { + this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', true); + this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', false); + } + this._lastKeyboardInput = newValue; + } else { + Log.Debug("Non-IME input change, sending new characters"); + const newValue = e.target.value; + + if (!this._lastKeyboardInput) { + this._keyboardInputReset(); + } + + const oldValue = this._lastKeyboardInput; + let newLen; + + try { + // Try to check caret position since whitespace at the end + // will not be considered by value.length in some browsers + newLen = Math.max(e.target.selectionStart, newValue.length); + } catch (err) { + // selectionStart is undefined in Google Chrome + newLen = newValue.length; + } + const oldLen = oldValue.length; + + let inputs = newLen - oldLen; + let backspaces = inputs < 0 ? -inputs : 0; + + // Compare the old string with the new to account for + // text-corrections or other input that modify existing text + for (let i = 0; i < Math.min(oldLen, newLen); i++) { + if (newValue.charAt(i) != oldValue.charAt(i)) { + inputs = newLen - i; + backspaces = oldLen - i; + break; + } + } + + // Send the key events + for (let i = 0; i < backspaces; i++) { + this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", true); + this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", false); + } + for (let i = newLen - inputs; i < newLen; i++) { + this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', true); + this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', false); + } + + // Control the text content length in the keyboardinput element + if (newLen > 2 * this._defaultKeyboardInputLen) { + this._keyboardInputReset(); + } else if (newLen < 1) { + // There always have to be some text in the keyboardinput + // element with which backspace can interact. + this._keyboardInputReset(); + // This sometimes causes the keyboard to disappear for a second + // but it is required for the android keyboard to recognize that + // text has been added to the field + e.target.blur(); + // This has to be ran outside of the input handler in order to work + setTimeout(e.target.focus.bind(e.target), 0); + } else { + this._lastKeyboardInput = newValue; + } + } + } + + _keyboardInputReset() { + this._touchInput.value = new Array(this._defaultKeyboardInputLen).join("_"); + this._lastKeyboardInput = this._touchInput.value; + } + _handleKeyDown(e) { const code = this._getKeyCode(e); let keysym = KeyboardUtil.getKeysym(e); + if (this._isIMEInteraction(e)) { + //skip event if IME related + Log.Debug("Skipping keydown, IME interaction, code: " + code + " keysym: " + keysym + " keycode: " + e.keyCode); + return; + } + // Windows doesn't have a proper AltGr, but handles it using // fake Ctrl+Alt. However the remote end might not be Windows, // so we need to merge those in to a single AltGr event. We @@ -220,10 +366,15 @@ export default class Keyboard { } _handleKeyUp(e) { - stopEvent(e); - const code = this._getKeyCode(e); + if (this._isIMEInteraction(e)) { + //skip IME related events + Log.Debug("Skipping keyup, IME interaction, code: " + code + " keycode: " + e.keyCode); + return; + } + stopEvent(e); + // We can't get a release in the middle of an AltGr sequence, so // abort that detection if (this._altGrArmed) { @@ -271,13 +422,56 @@ export default class Keyboard { Log.Debug("<< Keyboard.allKeysUp"); } + _isIMEInteraction(e) { + //input must come from touchinput (textarea) and ime must be enabled + if (e.target != this._touchInput || !this._enableIME) { return false; } + + //keyCode of 229 is IME composition + if (e.keyCode == 229) { + return true; + } + + //unfortunately, IME interactions can come through as events + //generally safe to ignore and let them come in as "input" events instead + //we can't do that with none character keys though + //Firefox does not seem to fire key events for IME interaction but Chrome does + //TODO: potentially skip this for Firefox browsers, needs more testing with different IME types + if (e.keyCode in imekeys) { + return true; + } + + return false; + } + // ===== PUBLIC METHODS ===== + focus() { + if (this._enableIME) { + this._touchInput.focus(); + } else { + this._screenInput.focus(); + } + } + + blur() { + if (this._enableIME) { + this._touchInput.blur(); + } else { + this._screenInput.blur(); + } + } + grab() { //Log.Debug(">> Keyboard.grab"); - this._target.addEventListener('keydown', this._eventHandlers.keydown); - this._target.addEventListener('keyup', this._eventHandlers.keyup); + this._screenInput.addEventListener('keydown', this._eventHandlers.keydown); + this._screenInput.addEventListener('keyup', this._eventHandlers.keyup); + + this._touchInput.addEventListener('keydown', this._eventHandlers.keydown); + this._touchInput.addEventListener('keyup', this._eventHandlers.keyup); + this._touchInput.addEventListener('compositionstart', this._eventHandlers.compositionstart); + this._touchInput.addEventListener('compositionend', this._eventHandlers.compositionend); + this._touchInput.addEventListener('input', this._eventHandlers.input); // Release (key up) if window loses focus window.addEventListener('blur', this._eventHandlers.blur); @@ -288,8 +482,15 @@ export default class Keyboard { ungrab() { //Log.Debug(">> Keyboard.ungrab"); - this._target.removeEventListener('keydown', this._eventHandlers.keydown); - this._target.removeEventListener('keyup', this._eventHandlers.keyup); + this._screenInput.removeEventListener('keydown', this._eventHandlers.keydown); + this._screenInput.removeEventListener('keyup', this._eventHandlers.keyup); + + this._touchInput.removeEventListener('keydown', this._eventHandlers.keydown); + this._touchInput.removeEventListener('keyup', this._eventHandlers.keyup); + this._touchInput.removeEventListener('compositionstart', this._eventHandlers.compositionstart); + this._touchInput.removeEventListener('compositionend', this._eventHandlers.compositionend); + this._touchInput.removeEventListener('input', this._eventHandlers.input); + window.removeEventListener('blur', this._eventHandlers.blur); // Release (key up) all keys that are in a down state diff --git a/core/rfb.js b/core/rfb.js index a42e724a..c71b0a68 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -70,7 +70,7 @@ const extendedClipboardActionNotify = 1 << 27; const extendedClipboardActionProvide = 1 << 28; export default class RFB extends EventTargetMixin { - constructor(target, urlOrChannel, options) { + constructor(target, touchInput, urlOrChannel, options) { if (!target) { throw new Error("Must specify target"); } @@ -244,7 +244,7 @@ export default class RFB extends EventTargetMixin { } this._display.onflush = this._onFlush.bind(this); - this._keyboard = new Keyboard(this._canvas); + this._keyboard = new Keyboard(this._canvas, touchInput); this._keyboard.onkeyevent = this._handleKeyEvent.bind(this); this._gestures = new GestureHandler(); @@ -333,6 +333,8 @@ export default class RFB extends EventTargetMixin { // ===== PROPERTIES ===== + get keyboard() { return this._keyboard; } + get clipboardBinary() { return this._clipboardMode; } set clipboardBinary(val) { this._clipboardMode = val; } @@ -745,11 +747,13 @@ export default class RFB extends EventTargetMixin { } focus() { - this._canvas.focus(); + this._keyboard.focus(); + //this._canvas.focus(); } blur() { - this._canvas.blur(); + this._keyboard.blur(); + //this._canvas.blur(); } clipboardPasteFrom(text) { diff --git a/core/util/browser.js b/core/util/browser.js index 39ca4468..23798f39 100644 --- a/core/util/browser.js +++ b/core/util/browser.js @@ -97,6 +97,10 @@ export function isSafari() { navigator.userAgent.indexOf('Chrome') === -1); } +export function isChromiumBased() { + return (!!window.chrome); +} + export function isFirefox() { return navigator && !!(/firefox/i).exec(navigator.userAgent); } diff --git a/vnc.html b/vnc.html index 4025bb6b..bbcf034f 100644 --- a/vnc.html +++ b/vnc.html @@ -202,6 +202,10 @@
    • +
  • +
    • +
  • +
  • @@ -444,7 +448,7 @@ -
    +
    @@ -454,7 +458,7 @@
    -
    +