Merge pull request #21 from kasmtech/feature/KASM-2001_mobile_keyboard

KASM-2001 Add keyboard controls panel

app/styles/base.css

@@ -971,6 +971,147 @@ select:active {
   display: none;
 }
 
+/* prevent selection */
+body {
+  -webkit-tap-highlight-color: transparent;
+  -webkit-touch-callout: none;
+  -webkit-user-select: none;
+  -khtml-user-select: none;
+  -moz-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+}
+
+/*
+ *
+ */
+.keyboard-controls {
+  display: none;
+  position: fixed;
+  bottom: 5%;
+  right: 5%;
+  z-index: 100000;
+  cursor: pointer;
+}
+
+.keyboard-controls .buttons {
+  display: none;
+  flex-direction: column;
+}
+
+.keyboard-controls .buttons .button {
+  border-radius: 0px;
+}
+
+.keyboard-controls .buttons .button:first-of-type {
+  border-top-left-radius: 6px;
+  border-top-right-radius: 6px;
+}
+
+.keyboard-controls .button {
+  display: inline-block;
+  width: 35px;
+  height: 35px;
+
+  background-color:rgb(24, 31, 71);
+  border: 6px rgb(24, 31, 71) solid;
+
+  border-radius: 6px;
+  outline: none;
+  user-select: none;
+}
+
+.keyboard-controls .button.ctrl {
+  background-image: url("../images/ctrl.svg");
+  background-size: contain;
+}    
+
+.keyboard-controls .button.alt {
+  background-image: url("../images/alt.svg");
+  background-size: contain;
+}
+
+.keyboard-controls .button.windows {
+  background-size: contain;
+  background-image: url("../images/windows.svg");
+}
+
+.keyboard-controls .button.tab {
+  background-size: contain;
+  background-image: url("../images/tab.svg");
+}
+
+.keyboard-controls .button.escape {
+  background-size: contain;
+  background-image: url("../images/esc.svg");
+}
+
+.keyboard-controls .button.ctrlaltdel {
+  background-size: contain;
+  background-image: url("../images/ctrlaltdel.svg");
+}
+
+.keyboard-controls .button.keyboard {
+  background-size: contain;
+  background-image: url("../images/keyboard.svg");
+}
+
+.keyboard-controls .button.selected {
+  filter: brightness(150%);
+}
+
+.keyboard-controls.is-visible {
+  display: flex;
+  flex-direction: column;
+}
+
+.keyboard-controls.is-open .button.handle {
+  border-top-left-radius: 0px;
+  border-top-right-radius: 0px;
+}
+
+.keyboard-controls.is-open .buttons {
+  display: flex;
+  flex-direction: column;
+
+  animation-name: showKeyboardControls;
+  animation-duration: 0.2s;
+  animation-timing-function: linear;
+  animation-fill-mode: both;
+}
+
+.keyboard-controls.was-open .buttons {
+  display: flex;
+  flex-direction: column;
+
+  animation-name: hideKeyboardControls;
+  animation-duration: 0.2s;
+  animation-timing-function: linear;
+  animation-fill-mode: both;
+}
+
+@keyframes showKeyboardControls {
+  0% {
+    transform: scale(1, 0.2);
+    transform-origin: bottom center;
+  }
+  100% {
+    transform: scale(1, 1);
+    transform-origin: bottom center;
+  }
+}
+
+@keyframes hideKeyboardControls {
+  0% {
+    transform: scale(1, 1);
+    transform-origin: bottom center;
+  }
+  100% {
+    transform: scale(1, 0);
+    transform-origin: bottom center;
+  }
+}
+
 /* ----------------------------------------
  * Media sizing
  * ----------------------------------------

app/ui.js

@@ -30,11 +30,19 @@ window.updateSetting = (name, value) => {
     }
 }
 
+window.showKeyboardControlsPanel = () => {
+    document.querySelector(".keyboard-controls").classList.add("is-visible");
+}
+
+window.hideKeyboardControlsPanel = () => {
+    document.querySelector(".keyboard-controls").classList.remove("is-visible");
+}
+
 import "core-js/stable";
 import "regenerator-runtime/runtime";
 import * as Log from '../core/util/logging.js';
 import _, { l10n } from './localization.js';
-import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox }
+import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS }
     from '../core/util/browser.js';
 import { setCapture, getPointerEvent } from '../core/util/events.js';
 import KeyTable from "../core/input/keysym.js";
@@ -125,6 +133,7 @@ const UI = {
         UI.initFullscreen();
 
         // Setup event handlers
+        UI.addKeyboardControlsPanelHandlers();
         UI.addControlbarHandlers();
         UI.addTouchSpecificHandlers();
         UI.addExtraKeysHandlers();
@@ -154,6 +163,14 @@ const UI = {
             UI.openConnectPanel();
         }
 
+        if ( !isWindows() && (
+                (window.parent.KASM_INITIAL_KEYBOARD_CONTROLS_MODE === "on") ||
+                (window.parent.KASM_INITIAL_KEYBOARD_CONTROLS_MODE === "auto" && isTouchDevice)
+            )
+        ) {
+            showKeyboardControlsPanel();
+        }
+
         return Promise.resolve(UI.rfb);
     },
 
@@ -272,6 +289,51 @@ const UI = {
 * EVENT HANDLERS
 * ------v------*/
 
+    addKeyboardControlsPanelHandlers() {
+        // panel dragging
+        interact(".keyboard-controls").draggable({
+            allowFrom: ".handle",
+            listeners: {
+                move: (e) => {
+                    const target = e.target;
+                    const x = (parseFloat(target.getAttribute("data-x")) || 0) + e.dx;
+                    const y = (parseFloat(target.getAttribute("data-y")) || 0) + e.dy;
+                    target.style.transform = `translate(${x}px, ${y}px)`;
+                    target.setAttribute("data-x", x);
+                    target.setAttribute("data-y", y);
+                },
+            },
+        });
+
+        // panel expanding
+        interact(".keyboard-controls .handle")
+        .pointerEvents({ holdDuration: 350 })
+        .on("hold", (e) => {
+            const buttonsEl = document.querySelector(".keyboard-controls");
+    
+            const isOpen = buttonsEl.classList.contains("is-open");
+            buttonsEl.classList.toggle("was-open", isOpen);
+            buttonsEl.classList.toggle("is-open", !isOpen);
+
+            setTimeout(() => buttonsEl.classList.remove("was-open"), 500);
+        });
+
+        // keyboard showing
+        interact(".keyboard-controls .handle").on("tap", (e) => {
+            if (e.dt < 150) {
+                UI.toggleVirtualKeyboard();
+            }
+        });
+
+        // panel buttons
+        interact(".keyboard-controls .button.ctrl").on("tap", UI.toggleCtrl);
+        interact(".keyboard-controls .button.alt").on("tap", UI.toggleAlt);
+        interact(".keyboard-controls .button.windows").on("tap", UI.toggleWindows);
+        interact(".keyboard-controls .button.tab").on("tap", UI.sendTab);
+        interact(".keyboard-controls .button.escape").on("tap", UI.sendEsc);
+        interact(".keyboard-controls .button.ctrlaltdel").on("tap", UI.sendCtrlAltDel);
+    },
+
     addControlbarHandlers() {
         document.getElementById("noVNC_control_bar")
             .addEventListener('mousemove', UI.activateControlbar);
@@ -1268,13 +1330,17 @@ const UI = {
         UI.rfb.addEventListener("capabilities", UI.updatePowerButton);
         UI.rfb.addEventListener("clipboard", UI.clipboardReceive);
         UI.rfb.addEventListener("bottleneck_stats", UI.bottleneckStatsRecieve);
-        document.addEventListener('mouseenter', UI.enterVNC);
+
-        document.addEventListener('mouseleave', UI.leaveVNC);
+        if (!isTouchDevice) {
-        document.addEventListener('blur', UI.blurVNC);
+            document.addEventListener('mouseenter', UI.enterVNC);
-        document.addEventListener('focus', UI.focusVNC);
+            document.addEventListener('mouseleave', UI.leaveVNC);
-        document.addEventListener('focusout', UI.focusoutVNC);
+            document.addEventListener('focusout', UI.focusoutVNC);
-        document.addEventListener('mousemove', UI.mouseMoveVNC);
+            document.addEventListener('mousemove', UI.mouseMoveVNC);
-        document.addEventListener('mousedown', UI.mouseDownVNC);
+            document.addEventListener('mousedown', UI.mouseDownVNC);
+            document.addEventListener('blur', UI.blurVNC);
+            document.addEventListener('focus', UI.focusVNC);
+        }
+
         UI.rfb.addEventListener("bell", UI.bell);
         UI.rfb.addEventListener("desktopname", UI.updateDesktopName);
         UI.rfb.translateShortcuts = UI.getSetting('translate_shortcuts');
@@ -1849,8 +1915,6 @@ const UI = {
     },
 
     showVirtualKeyboard() {
-        if (!isTouchDevice) return;
-
         const input = document.getElementById('noVNC_keyboardinput');
 
         if (document.activeElement == input) return;
@@ -1864,11 +1928,17 @@ const UI = {
         } catch (err) {
             // setSelectionRange is undefined in Google Chrome
         }
+
+        // ensure that the hidden input used for showing the virutal keyboard
+        // does not steal focus if the user has closed it manually
+        document.querySelector("canvas").addEventListener("touchstart", () => {
+            if (document.activeElement === input) {
+                input.blur();
+            }
+        }, { once: true });
     },
 
     hideVirtualKeyboard() {
-        if (!isTouchDevice) return;
-
         const input = document.getElementById('noVNC_keyboardinput');
 
         if (document.activeElement != input) return;
@@ -2025,6 +2095,14 @@ const UI = {
             .classList.add("noVNC_selected");
     },
 
+    disableSoftwareKeyboard() {
+        document.querySelector("#noVNC_keyboard_button").disabled = true;
+    },
+
+    enableSoftwareKeyboard() {
+        document.querySelector("#noVNC_keyboard_button").disabled = false;
+    },
+
     closeExtraKeys() {
         document.getElementById('noVNC_modifiers')
             .classList.remove("noVNC_open");
@@ -2033,8 +2111,7 @@ const UI = {
     },
 
     toggleExtraKeys() {
-        if (document.getElementById('noVNC_modifiers')
+        if (document.getElementById('noVNC_modifiers').classList.contains("noVNC_open")) {
-            .classList.contains("noVNC_open")) {
             UI.closeExtraKeys();
         } else  {
             UI.openExtraKeys();
@@ -2058,6 +2135,8 @@ const UI = {
             UI.sendKey(KeyTable.XK_Control_L, "ControlLeft", true);
             btn.classList.add("noVNC_selected");
         }
+
+        document.querySelector(".keyboard-controls .button.ctrl").classList.toggle("selected");
     },
 
     toggleWindows() {
@@ -2069,6 +2148,8 @@ const UI = {
             UI.sendKey(KeyTable.XK_Super_L, "MetaLeft", true);
             btn.classList.add("noVNC_selected");
         }
+
+        document.querySelector(".keyboard-controls .button.windows").classList.toggle("selected");
     },
 
     toggleAlt() {
@@ -2080,6 +2161,8 @@ const UI = {
             UI.sendKey(KeyTable.XK_Alt_L, "AltLeft", true);
             btn.classList.add("noVNC_selected");
         }
+
+        document.querySelector(".keyboard-controls .button.alt").classList.toggle("selected");
     },
 
     sendCtrlAltDel() {

core/rfb.js

@@ -11,7 +11,7 @@ import { toUnsigned32bit, toSigned32bit } from './util/int.js';
 import * as Log from './util/logging.js';
 import { encodeUTF8, decodeUTF8 } from './util/strings.js';
 import { hashUInt8Array } from './util/int.js';
-import { dragThreshold, supportsCursorURIs, isTouchDevice, isWindows, isMac } from './util/browser.js';
+import { dragThreshold, supportsCursorURIs, isTouchDevice, isWindows, isMac, isIOS } from './util/browser.js';
 import { clientToElement } from './util/element.js';
 import { setCapture } from './util/events.js';
 import EventTargetMixin from './util/eventtarget.js';
@@ -192,6 +192,7 @@ export default class RFB extends EventTargetMixin {
 
         // Bound event handlers
         this._eventHandlers = {
+            updateHiddenKeyboard: this._updateHiddenKeyboard.bind(this),
             focusCanvas: this._focusCanvas.bind(this),
             windowResize: this._windowResize.bind(this),
             handleMouse: this._handleMouse.bind(this),
@@ -894,6 +895,15 @@ export default class RFB extends EventTargetMixin {
         this._canvas.addEventListener("mousedown", this._eventHandlers.focusCanvas);
         this._canvas.addEventListener("touchstart", this._eventHandlers.focusCanvas);
 
+        // In order for the keyboard to not occlude the input being edited
+        // we move the hidden input we use for triggering the keyboard to the last click
+        // position which should trigger a page being moved down enough
+        // to show the input. On Android the whole website gets resized so we don't
+        // have to do anything.
+        if (isIOS()) {
+            this._canvas.addEventListener("touchend", this._eventHandlers.updateHiddenKeyboard);
+        }
+
         // Mouse events
         this._canvas.addEventListener('mousedown', this._eventHandlers.handleMouse);
         this._canvas.addEventListener('mouseup', this._eventHandlers.handleMouse);
@@ -948,6 +958,12 @@ export default class RFB extends EventTargetMixin {
         Log.Debug("<< RFB.disconnect");
     }
 
+    _updateHiddenKeyboard(event) {
+        // On iOS 15 the navigation bar is at the bottom so we need to account for it
+        const y = Math.max(0, event.pageY - 50);
+        document.querySelector("#noVNC_keyboardinput").style.top = `${y}px`;
+    }
+
     _focusCanvas(event) {
         // Hack:
         // On most mobile phones it's only possible to play audio
@@ -955,7 +971,7 @@ export default class RFB extends EventTargetMixin {
         // impossible to listen for touch events on child frames (only on mobile phones)
         // we delegate the audio unlocking to the parent window.
         if (window.parent && !window.parent.KASM_AUDIO_UNLOCKED) {
-            window.parent.unlockAudio();
+            window.parent.unlockAudio && window.parent.unlockAudio();
         }
 
         if (!this.focusOnClick) {

vnc.html

@@ -47,6 +47,8 @@
     <link rel="apple-touch-icon" sizes="120x120" type="image/png" href="app/images/icons/368_kasm_logo_only_120x120.png">
     <link rel="apple-touch-icon" sizes="152x152" type="image/png" href="app/images/icons/368_kasm_logo_only_152x152.png">
 
+    <script src="vendor/interact.min.js"></script>
+
     <!-- Stylesheets -->
     <!--link rel="stylesheet" href="app/styles/base.css">
 
@@ -84,6 +86,31 @@
         Loading statistics...
     </div>
 
+    <div class="noVNC_vcenter">
+        <div id="noVNC_modifiers" class="noVNC_panel">
+            <input type="image" alt="Keyboard" src="app/images/keyboard.svg"
+                id="noVNC_keyboard_button" class="noVNC_button" title="Show Keyboard">
+            <input type="image" alt="Ctrl" src="app/images/ctrl.svg"
+                id="noVNC_toggle_ctrl_button" class="noVNC_button"
+                title="Toggle Ctrl">
+            <input type="image" alt="Alt" src="app/images/alt.svg"
+                id="noVNC_toggle_alt_button" class="noVNC_button"
+                title="Toggle Alt">
+            <input type="image" alt="Windows" src="app/images/windows.svg"
+                id="noVNC_toggle_windows_button" class="noVNC_button"
+                title="Toggle Windows">
+            <input type="image" alt="Tab" src="app/images/tab.svg"
+                id="noVNC_send_tab_button" class="noVNC_button"
+                title="Send Tab">
+            <input type="image" alt="Esc" src="app/images/esc.svg"
+                id="noVNC_send_esc_button" class="noVNC_button"
+                title="Send Escape">
+            <input type="image" alt="Ctrl+Alt+Del" src="app/images/ctrlaltdel.svg"
+                id="noVNC_send_ctrl_alt_del_button" class="noVNC_button"
+                title="Send Ctrl-Alt-Del">
+        </div>
+    </div>
+
     <!-- noVNC Control Bar -->
     <div id="noVNC_control_bar_anchor" class="noVNC_vcenter">
 
@@ -100,37 +127,10 @@
                 title="Move/Drag Viewport">
 
             <!--noVNC Touch Device only buttons-->
-            <div id="noVNC_mobile_buttons">
-                <input type="image" alt="Keyboard" src="app/images/keyboard.svg"
-                    id="noVNC_keyboard_button" class="noVNC_button" title="Show Keyboard">
-            </div>
-
             <!-- Extra manual keys -->
             <input type="image" alt="Extra keys" src="app/images/toggleextrakeys.svg"
                 id="noVNC_toggle_extra_keys_button" class="noVNC_button"
                 title="Show Extra Keys">
-            <div class="noVNC_vcenter">
-            <div id="noVNC_modifiers" class="noVNC_panel">
-                <input type="image" alt="Ctrl" src="app/images/ctrl.svg"
-                    id="noVNC_toggle_ctrl_button" class="noVNC_button"
-                    title="Toggle Ctrl">
-                <input type="image" alt="Alt" src="app/images/alt.svg"
-                    id="noVNC_toggle_alt_button" class="noVNC_button"
-                    title="Toggle Alt">
-                <input type="image" alt="Windows" src="app/images/windows.svg"
-                    id="noVNC_toggle_windows_button" class="noVNC_button"
-                    title="Toggle Windows">
-                <input type="image" alt="Tab" src="app/images/tab.svg"
-                    id="noVNC_send_tab_button" class="noVNC_button"
-                    title="Send Tab">
-                <input type="image" alt="Esc" src="app/images/esc.svg"
-                    id="noVNC_send_esc_button" class="noVNC_button"
-                    title="Send Escape">
-                <input type="image" alt="Ctrl+Alt+Del" src="app/images/ctrlaltdel.svg"
-                    id="noVNC_send_ctrl_alt_del_button" class="noVNC_button"
-                    title="Send Ctrl-Alt-Del">
-            </div>
-            </div>
 
             <!-- Shutdown/Reboot -->
             <input type="image" alt="Shutdown/Reboot" src="app/images/power.svg"
@@ -443,5 +443,18 @@
         <source src="app/sounds/bell.oga" type="audio/ogg">
         <source src="app/sounds/bell.mp3" type="audio/mpeg">
     </audio>
+
+    <div class="keyboard-controls">
+        <div class="buttons">
+            <div class="button ctrl"></div>
+            <div class="button alt"></div>
+            <div class="button windows"></div>
+            <div class="button tab"></div>
+            <div class="button escape"></div>
+            <div class="button ctrlaltdel"></div>
+        </div>
+        
+        <div class="button keyboard handle"></div>
+    </div>
  </body>
-</html>
+</html>