jcarr
/
noVNC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/kasm 2335 ime support 2 (#27) 

IME support, refactored keyboard input
This commit is contained in:
Matt McClaskey 2022-03-25 15:03:19 -04:00 committed by GitHub
parent df9c9d0d96
commit 385a1f99b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 344 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
app/styles/base.css
View File

@ -936,15 +936,15 @@ select:active {
}
#noVNC_keyboardinput {
width: 1px;
height: 1px;
background-color: #fff;
color: #fff;
width: 0px;
height: 0px;
background-color: #fff0;
color: rgba(5, 5, 5, 0);
border: 0;
position: absolute;
left: -40px;
left: 35%;
top: 40%;
z-index: -1;
ime-mode: disabled;
}
/*Default noVNC logo.*/
@ -1021,6 +1021,11 @@ body {
user-select: none;
}
#noVNC_keyboard_control .noVNC_selected {
background-color:rgb(15, 36, 153);
border: 6px rgb(15, 36, 153) solid;
}
.keyboard-controls .button.ctrl {
background-image: url("../images/ctrl.svg");
background-size: contain;

167
app/ui.js
View File

@ -65,8 +65,6 @@ const UI = {
controlbarMouseDownClientY: 0,
controlbarMouseDownOffsetY: 0,
lastKeyboardinput: null,
defaultKeyboardinputLen: 100,
needToCheckClipboardChange: false,
inhibitReconnect: true,
@ -135,10 +133,6 @@ const UI = {
UI.addSettingsHandlers();
document.getElementById("noVNC_status")
.addEventListener('click', UI.hideStatus);
// Bootstrap fallback input handler
UI.keyboardinputReset();
UI.openControlbar();
UI.updateVisualState('init');
@ -245,6 +239,9 @@ const UI = {
UI.initSetting('prefer_local_cursor', true);
UI.initSetting('toggle_control_panel', false);
UI.initSetting('enable_perf_stats', false);
UI.initSetting('virtual_keyboard_visible', false);
UI.initSetting('enable_ime', false)
UI.toggleKeyboardControls();
if (WebUtil.isInsideKasmVDI()) {
UI.initSetting('clipboard_up', false);
@ -371,12 +368,6 @@ const UI = {
addTouchSpecificHandlers() {
document.getElementById("noVNC_keyboard_button")
.addEventListener('click', UI.toggleVirtualKeyboard);
UI.touchKeyboard = new Keyboard(document.getElementById('noVNC_keyboardinput'));
UI.touchKeyboard.onkeyevent = UI.keyEvent;
UI.touchKeyboard.grab();
document.getElementById("noVNC_keyboardinput")
.addEventListener('input', UI.keyInput);
document.getElementById("noVNC_keyboardinput")
.addEventListener('focus', UI.onfocusVirtualKeyboard);
document.getElementById("noVNC_keyboardinput")
@ -536,6 +527,10 @@ const UI = {
UI.addSettingChangeHandler('clipboard_seamless');
UI.addSettingChangeHandler('clipboard_up');
UI.addSettingChangeHandler('clipboard_down');
UI.addSettingChangeHandler('virtual_keyboard_visible');
UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls);
UI.addSettingChangeHandler('enable_ime');
UI.addSettingChangeHandler('enable_ime', UI.toggleIMEMode);
},
addFullscreenHandlers() {
@ -1297,7 +1292,9 @@ const UI = {
}
url += '/' + path;
UI.rfb = new RFB(document.getElementById('noVNC_container'), url,
UI.rfb = new RFB(document.getElementById('noVNC_container'),
document.getElementById('noVNC_keyboardinput'),
url,
{ shared: UI.getSetting('shared'),
repeaterID: UI.getSetting('repeaterID'),
credentials: { password: password } });
@ -1335,6 +1332,7 @@ const UI = {
UI.rfb.clipboardUp = UI.getSetting('clipboard_up');
UI.rfb.clipboardDown = UI.getSetting('clipboard_down');
UI.rfb.clipboardSeamless = UI.getSetting('clipboard_seamless');
UI.rfb.keyboard.enableIME = UI.getSetting('enable_ime');
UI.rfb.clipboardBinary = supportsBinaryClipboard() && UI.rfb.clipboardSeamless;
//Only explicitly request permission to clipboard on browsers that support binary clipboard access
@ -1544,6 +1542,30 @@ const UI = {
UI.forceSetting('video_quality', parseInt(event.data.value), false);
UI.updateQuality();
break;
case 'show_keyboard_controls':
if (!UI.getSetting('virtual_keyboard_visible')) {
UI.forceSetting('virtual_keyboard_visible', true, false);
UI.showKeyboardControls();
}
break;
case 'hide_keyboard_controls':
if (UI.getSetting('virtual_keyboard_visible')) {
UI.forceSetting('virtual_keyboard_visible', true, false);
UI.hideKeyboardControls();
}
break;
case 'enable_ime_mode':
if (!UI.getSetting('enable_ime')) {
UI.forceSetting('enable_ime', true, false);
UI.toggleIMEMode();
}
break;
case 'disable_ime_mode':
if (UI.getSetting('enable_ime')) {
UI.forceSetting('enable_ime', false, false);
UI.toggleIMEMode();
}
break;
}
}
},
@ -1881,18 +1903,41 @@ const UI = {
UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts');
},
toggleKeyboardControls() {
if (UI.getSetting('virtual_keyboard_visible')) {
UI.showKeyboardControls();
} else {
UI.hideKeyboardControls();
}
},
toggleIMEMode() {
if (UI.rfb) {
if (UI.getSetting('enable_ime')) {
UI.rfb.keyboard.enableIME = true;
} else {
UI.rfb.keyboard.enableIME = false;
}
}
},
showKeyboardControls() {
document.querySelector(".keyboard-controls").classList.add("is-visible");
document.getElementById('noVNC_keyboard_control').classList.add("is-visible");
},
hideKeyboardControls() {
document.querySelector(".keyboard-controls").classList.remove("is-visible");
document.getElementById('noVNC_keyboard_control').classList.remove("is-visible");
},
showVirtualKeyboard() {
const input = document.getElementById('noVNC_keyboardinput');
if (document.activeElement == input) return;
if (document.activeElement == input || !UI.rfb) return;
if (UI.getSetting('virtual_keyboard_visible')) {
document.getElementById('noVNC_keyboard_control_handle')
.classList.add("noVNC_selected");
}
input.focus();
@ -1916,7 +1961,12 @@ const UI = {
hideVirtualKeyboard() {
const input = document.getElementById('noVNC_keyboardinput');
if (document.activeElement != input) return;
if (document.activeElement != input || !UI.rfb) return;
if (UI.getSetting('virtual_keyboard_visible')) {
document.getElementById('noVNC_keyboard_control_handle')
.classList.remove("noVNC_selected");
}
input.blur();
},
@ -1941,6 +1991,12 @@ const UI = {
onblurVirtualKeyboard(event) {
document.getElementById('noVNC_keyboard_button')
.classList.remove("noVNC_selected");
if (UI.getSetting('virtual_keyboard_visible')) {
document.getElementById('noVNC_keyboard_control_handle')
.classList.remove("noVNC_selected");
}
if (UI.rfb) {
UI.rfb.focusOnClick = true;
}
@ -1974,83 +2030,6 @@ const UI = {
event.preventDefault();
},
keyboardinputReset() {
const kbi = document.getElementById('noVNC_keyboardinput');
kbi.value = new Array(UI.defaultKeyboardinputLen).join("_");
UI.lastKeyboardinput = kbi.value;
},
keyEvent(keysym, code, down) {
if (!UI.rfb) return;
UI.rfb.sendKey(keysym, code, down);
},
// When normal keyboard events are left uncought, use the input events from
// the keyboardinput element instead and generate the corresponding key events.
// This code is required since some browsers on Android are inconsistent in
// sending keyCodes in the normal keyboard events when using on screen keyboards.
keyInput(event) {
if (!UI.rfb) return;
const newValue = event.target.value;
if (!UI.lastKeyboardinput) {
UI.keyboardinputReset();
}
const oldValue = UI.lastKeyboardinput;
let newLen;
try {
// Try to check caret position since whitespace at the end
// will not be considered by value.length in some browsers
newLen = Math.max(event.target.selectionStart, newValue.length);
} catch (err) {
// selectionStart is undefined in Google Chrome
newLen = newValue.length;
}
const oldLen = oldValue.length;
let inputs = newLen - oldLen;
let backspaces = inputs < 0 ? -inputs : 0;
// Compare the old string with the new to account for
// text-corrections or other input that modify existing text
for (let i = 0; i < Math.min(oldLen, newLen); i++) {
if (newValue.charAt(i) != oldValue.charAt(i)) {
inputs = newLen - i;
backspaces = oldLen - i;
break;
}
}
// Send the key events
for (let i = 0; i < backspaces; i++) {
UI.rfb.sendKey(KeyTable.XK_BackSpace, "Backspace");
}
for (let i = newLen - inputs; i < newLen; i++) {
UI.rfb.sendKey(keysyms.lookup(newValue.charCodeAt(i)));
}
// Control the text content length in the keyboardinput element
if (newLen > 2 * UI.defaultKeyboardinputLen) {
UI.keyboardinputReset();
} else if (newLen < 1) {
// There always have to be some text in the keyboardinput
// element with which backspace can interact.
UI.keyboardinputReset();
// This sometimes causes the keyboard to disappear for a second
// but it is required for the android keyboard to recognize that
// text has been added to the field
event.target.blur();
// This has to be ran outside of the input handler in order to work
setTimeout(event.target.focus.bind(event.target), 0);
} else {
UI.lastKeyboardinput = newValue;
}
},
/* ------^-------
* /KEYBOARD
* ==============

32
core/input/imekeys.js Normal file
View File

@ -0,0 +1,32 @@
/*
* KasmVNC: HTML5 VNC client
* Copyright (C) 2022 Kasm Technologies Inc
* Licensed under MPL 2.0 or any later version (see LICENSE.txt)
*/
/*
* Keys that could be interaction with IME input
*/
export default {
0x30: 'Digit0',
0x31: 'Digit1',
0x32: 'Digit2',
0x33: 'Digit3',
0x34: 'Digit4',
0x35: 'Digit5',
0x36: 'Digit6',
0x37: 'Digit7',
0x38: 'Digit8',
0x39: 'Digit9',
0x60: 'Numpad0',
0x61: 'Numpad1',
0x62: 'Numpad2',
0x63: 'Numpad3',
0x64: 'Numpad4',
0x65: 'Numpad5',
0x66: 'Numpad6',
0x67: 'Numpad7',
0x68: 'Numpad8',
0x69: 'Numpad9'
};

219
core/input/keyboard.js
View File

@ -8,16 +8,20 @@ import * as Log from '../util/logging.js';
import { stopEvent } from '../util/events.js';
import * as KeyboardUtil from "./util.js";
import KeyTable from "./keysym.js";
import keysyms from "./keysymdef.js";
import imekeys from "./imekeys.js";
import * as browser from "../util/browser.js";
import UI from '../../app/ui.js';
import { isChromiumBased } from '../util/browser.js';
//
// Keyboard event handler
//
export default class Keyboard {
constructor(target) {
this._target = target || null;
constructor(screenInput, touchInput) {
this._screenInput = screenInput;
this._touchInput = touchInput;
this._keyDownList = {}; // List of depressed keys
// (even if they are happy)
@ -28,11 +32,28 @@ export default class Keyboard {
'keyup': this._handleKeyUp.bind(this),
'keydown': this._handleKeyDown.bind(this),
'blur': this._allKeysUp.bind(this),
'compositionstart' : this._handleCompositionStart.bind(this),
'compositionend' : this._handleCompositionEnd.bind(this),
'input' : this._handleInput.bind(this)
};
// ===== EVENT HANDLERS =====
this.onkeyevent = () => {}; // Handler for key press/release
this._enableIME = false;
this._imeHold = false;
this._imeInProgress = false;
this._lastKeyboardInput = null;
this._defaultKeyboardInputLen = 100;
this._keyboardInputReset();
}
// ===== PUBLIC METHODS =====
get enableIME() { return this._enableIME; }
set enableIME(val) {
this._enableIME = val;
this.focus();
}
// ===== PRIVATE METHODS =====
@ -95,10 +116,135 @@ export default class Keyboard {
return 'Unidentified';
}
_handleCompositionStart(e) {
Log.Debug("composition started");
if (this._enableIME) {
this._imeHold = true;
this._imeInProgress = true;
}
}
_handleCompositionEnd(e) {
Log.Debug("Composition ended");
if (this._enableIME) { this._imeInProgress = false; }
if (isChromiumBased()) {
this._imeHold = false;
}
}
_handleInput(e) {
//input event occurs only when keyup keydown events don't prevent default
//IME events will make this happen, for example
//IME changes can back out old characters and replace, thus send differential if IME
//otherwise send new characters
if (this._enableIME && this._imeHold) {
Log.Debug("IME input change, sending differential");
if (!this._imeInProgress) {
this._imeHold = false; //Firefox fires compisitionend before last input change
}
const oldValue = this._lastKeyboardInput;
const newValue = e.target.value;
let diff_start = 0;
//find position where difference starts
for (let i = 0; i < Math.min(oldValue.length, newValue.length); i++) {
if (newValue.charAt(i) != oldValue.charAt(i)) {
break;
}
diff_start++;
}
//send backspaces if needed
for (let bs = oldValue.length - diff_start; bs > 0; bs--) {
this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", true);
this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", false);
}
//send new keys
for (let i = diff_start; i < newValue.length; i++) {
this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', true);
this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', false);
}
this._lastKeyboardInput = newValue;
} else {
Log.Debug("Non-IME input change, sending new characters");
const newValue = e.target.value;
if (!this._lastKeyboardInput) {
this._keyboardInputReset();
}
const oldValue = this._lastKeyboardInput;
let newLen;
try {
// Try to check caret position since whitespace at the end
// will not be considered by value.length in some browsers
newLen = Math.max(e.target.selectionStart, newValue.length);
} catch (err) {
// selectionStart is undefined in Google Chrome
newLen = newValue.length;
}
const oldLen = oldValue.length;
let inputs = newLen - oldLen;
let backspaces = inputs < 0 ? -inputs : 0;
// Compare the old string with the new to account for
// text-corrections or other input that modify existing text
for (let i = 0; i < Math.min(oldLen, newLen); i++) {
if (newValue.charAt(i) != oldValue.charAt(i)) {
inputs = newLen - i;
backspaces = oldLen - i;
break;
}
}
// Send the key events
for (let i = 0; i < backspaces; i++) {
this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", true);
this._sendKeyEvent(KeyTable.XK_BackSpace, "Backspace", false);
}
for (let i = newLen - inputs; i < newLen; i++) {
this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', true);
this._sendKeyEvent(keysyms.lookup(newValue.charCodeAt(i)), 'Unidentified', false);
}
// Control the text content length in the keyboardinput element
if (newLen > 2 * this._defaultKeyboardInputLen) {
this._keyboardInputReset();
} else if (newLen < 1) {
// There always have to be some text in the keyboardinput
// element with which backspace can interact.
this._keyboardInputReset();
// This sometimes causes the keyboard to disappear for a second
// but it is required for the android keyboard to recognize that
// text has been added to the field
e.target.blur();
// This has to be ran outside of the input handler in order to work
setTimeout(e.target.focus.bind(e.target), 0);
} else {
this._lastKeyboardInput = newValue;
}
}
}
_keyboardInputReset() {
this._touchInput.value = new Array(this._defaultKeyboardInputLen).join("_");
this._lastKeyboardInput = this._touchInput.value;
}
_handleKeyDown(e) {
const code = this._getKeyCode(e);
let keysym = KeyboardUtil.getKeysym(e);
if (this._isIMEInteraction(e)) {
//skip event if IME related
Log.Debug("Skipping keydown, IME interaction, code: " + code + " keysym: " + keysym + " keycode: " + e.keyCode);
return;
}
// Windows doesn't have a proper AltGr, but handles it using
// fake Ctrl+Alt. However the remote end might not be Windows,
// so we need to merge those in to a single AltGr event. We
@ -220,10 +366,15 @@ export default class Keyboard {
}
_handleKeyUp(e) {
stopEvent(e);
const code = this._getKeyCode(e);
if (this._isIMEInteraction(e)) {
//skip IME related events
Log.Debug("Skipping keyup, IME interaction, code: " + code + " keycode: " + e.keyCode);
return;
}
stopEvent(e);
// We can't get a release in the middle of an AltGr sequence, so
// abort that detection
if (this._altGrArmed) {
@ -271,13 +422,56 @@ export default class Keyboard {
Log.Debug("<< Keyboard.allKeysUp");
}
_isIMEInteraction(e) {
//input must come from touchinput (textarea) and ime must be enabled
if (e.target != this._touchInput || !this._enableIME) { return false; }
//keyCode of 229 is IME composition
if (e.keyCode == 229) {
return true;
}
//unfortunately, IME interactions can come through as events
//generally safe to ignore and let them come in as "input" events instead
//we can't do that with none character keys though
//Firefox does not seem to fire key events for IME interaction but Chrome does
//TODO: potentially skip this for Firefox browsers, needs more testing with different IME types
if (e.keyCode in imekeys) {
return true;
}
return false;
}
// ===== PUBLIC METHODS =====
focus() {
if (this._enableIME) {
this._touchInput.focus();
} else {
this._screenInput.focus();
}
}
blur() {
if (this._enableIME) {
this._touchInput.blur();
} else {
this._screenInput.blur();
}
}
grab() {
//Log.Debug(">> Keyboard.grab");
this._target.addEventListener('keydown', this._eventHandlers.keydown);
this._target.addEventListener('keyup', this._eventHandlers.keyup);
this._screenInput.addEventListener('keydown', this._eventHandlers.keydown);
this._screenInput.addEventListener('keyup', this._eventHandlers.keyup);
this._touchInput.addEventListener('keydown', this._eventHandlers.keydown);
this._touchInput.addEventListener('keyup', this._eventHandlers.keyup);
this._touchInput.addEventListener('compositionstart', this._eventHandlers.compositionstart);
this._touchInput.addEventListener('compositionend', this._eventHandlers.compositionend);
this._touchInput.addEventListener('input', this._eventHandlers.input);
// Release (key up) if window loses focus
window.addEventListener('blur', this._eventHandlers.blur);
@ -288,8 +482,15 @@ export default class Keyboard {
ungrab() {
//Log.Debug(">> Keyboard.ungrab");
this._target.removeEventListener('keydown', this._eventHandlers.keydown);
this._target.removeEventListener('keyup', this._eventHandlers.keyup);
this._screenInput.removeEventListener('keydown', this._eventHandlers.keydown);
this._screenInput.removeEventListener('keyup', this._eventHandlers.keyup);
this._touchInput.removeEventListener('keydown', this._eventHandlers.keydown);
this._touchInput.removeEventListener('keyup', this._eventHandlers.keyup);
this._touchInput.removeEventListener('compositionstart', this._eventHandlers.compositionstart);
this._touchInput.removeEventListener('compositionend', this._eventHandlers.compositionend);
this._touchInput.removeEventListener('input', this._eventHandlers.input);
window.removeEventListener('blur', this._eventHandlers.blur);
// Release (key up) all keys that are in a down state

12
core/rfb.js
View File

@ -70,7 +70,7 @@ const extendedClipboardActionNotify = 1 << 27;
const extendedClipboardActionProvide = 1 << 28;
export default class RFB extends EventTargetMixin {
constructor(target, urlOrChannel, options) {
constructor(target, touchInput, urlOrChannel, options) {
if (!target) {
throw new Error("Must specify target");
}
@ -244,7 +244,7 @@ export default class RFB extends EventTargetMixin {
}
this._display.onflush = this._onFlush.bind(this);
this._keyboard = new Keyboard(this._canvas);
this._keyboard = new Keyboard(this._canvas, touchInput);
this._keyboard.onkeyevent = this._handleKeyEvent.bind(this);
this._gestures = new GestureHandler();
@ -333,6 +333,8 @@ export default class RFB extends EventTargetMixin {
// ===== PROPERTIES =====
get keyboard() { return this._keyboard; }
get clipboardBinary() { return this._clipboardMode; }
set clipboardBinary(val) { this._clipboardMode = val; }
@ -745,11 +747,13 @@ export default class RFB extends EventTargetMixin {
}
focus() {
this._canvas.focus();
this._keyboard.focus();
//this._canvas.focus();
}
blur() {
this._canvas.blur();
this._keyboard.blur();
//this._canvas.blur();
}
clipboardPasteFrom(text) {

4
core/util/browser.js
View File

@ -97,6 +97,10 @@ export function isSafari() {
navigator.userAgent.indexOf('Chrome') === -1);
}
export function isChromiumBased() {
return (!!window.chrome);
}
export function isFirefox() {
return navigator && !!(/firefox/i).exec(navigator.userAgent);
}

8
vnc.html
View File

@ -202,6 +202,10 @@
<label><input id="noVNC_setting_enable_webp" type="checkbox" /> Enable WebP Compression</label></li>
<li>
<label><input id="noVNC_setting_enable_perf_stats" type="checkbox" /> Enable Performance Stats</label></li>
<li>
<label><input id="noVNC_setting_enable_ime" type="checkbox" /> IME Input Mode</label></li>
<li>
<label><input id="noVNC_setting_virtual_keyboard_visible" type="checkbox" /> Show Virtual Keyboard Control</label></li>
<li>
<label><input id="noVNC_setting_toggle_control_panel" type="checkbox" /> Toggle Control Panel via Keystrokes</label></li>
<li>
@ -444,7 +448,7 @@
<source src="app/sounds/bell.mp3" type="audio/mpeg">
</audio>
<div class="keyboard-controls">
<div id="noVNC_keyboard_control" class="keyboard-controls">
<div class="buttons">
<div class="button ctrl"></div>
<div class="button alt"></div>
@ -454,7 +458,7 @@
<div class="button ctrlaltdel"></div>
</div>
<div class="button keyboard handle"></div>
<div id="noVNC_keyboard_control_handle" class="button keyboard handle"></div>
</div>
</body>
</html>