Udp Initial MVP (#32)

* Initial UDP code

* refactored UDP reassembly

* WIP Added udp decoder

* WIP cleanup

* Port multiplexing

* UDP upgrade/downgrade

* UDP packets now have a hash

* Correct udp.js this.numColors bug

* auto upgrade to udp

* re-apply webpack

* Code cleanup, optimize clearing udpBuffer

Co-authored-by: Lauri Kasanen <cand@gmx.com>
Co-authored-by: matt <matt@kasmweb.com>

core/decoders/udp.js

@@ -0,0 +1,288 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2019 The noVNC Authors
+ * (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca)
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+import * as Log from '../util/logging.js';
+import Inflator from "../inflator.js";
+
+export default class UDPDecoder {
+    constructor() {
+        this._filter = null;
+        this._palette = new Uint8Array(1024);  // 256 * 4 (max palette size * max bytes-per-pixel)
+
+        this._zlibs = [];
+        for (let i = 0; i < 4; i++) {
+            this._zlibs[i] = new Inflator();
+        }
+    }
+
+    decodeRect(x, y, width, height, data, display, depth) {
+        let ctl = data[12];
+        ctl = ctl >> 4;
+
+        let ret;
+
+        if (ctl === 0x08) {
+            ret = this._fillRect(x, y, width, height,
+                data, display, depth);
+        } else if (ctl === 0x09) {
+            ret = this._jpegRect(x, y, width, height,
+                data, display, depth);
+        } else if (ctl === 0x0A) {
+            ret = this._pngRect(x, y, width, height,
+                data, display, depth);
+        } else if ((ctl & 0x08) == 0) {
+            ret = this._basicRect(ctl, x, y, width, height,
+                data, display, depth);
+        } else if (ctl === 0x0B) {
+            ret = this._webpRect(x, y, width, height,
+                data, display, depth);
+        } else {
+            throw new Error("Illegal udp compression received (ctl: " +
+                ctl + ")");
+        }
+
+        return ret;
+    }
+
+    _fillRect(x, y, width, height, data, display, depth) {
+
+        display.fillRect(x, y, width, height,
+            [data[13], data[14], data[15]], false);
+
+        return true;
+    }
+
+    _jpegRect(x, y, width, height, data, display, depth) {
+        let img = this._readData(data);
+        if (img === null) {
+            return false;
+        }
+
+        display.imageRect(x, y, width, height, "image/jpeg", img);
+
+        return true;
+    }
+
+    _webpRect(x, y, width, height, data, display, depth) {
+        let img = this._readData(data);
+        if (img === null) {
+            return false;
+        }
+
+        display.imageRect(x, y, width, height, "image/webp", img);
+
+        return true;
+    }
+
+    _pngRect(x, y, width, height, data, display, depth) {
+        //throw new Error("PNG received in UDP rect");
+        Log.Error("PNG received in UDP rect");
+    }
+
+    _basicRect(ctl, x, y, width, height, data, display, depth) {
+        let zlibs_flags = data[12];
+        // Reset streams if the server requests it
+        for (let i = 0; i < 4; i++) {
+            if ((zlibs_flags >> i) & 1) {
+                this._zlibs[i].reset();
+                Log.Info("Reset zlib stream " + i);
+            }
+        }
+
+        let filter = data[13];
+        let data_index = 14;
+        let streamId = ctl & 0x3;
+        if (!(ctl & 0x4)) {
+            // Implicit CopyFilter
+            filter = 0;
+            data_index = 13;
+        }
+
+        let ret;
+
+        switch (filter) {
+            case 0: // CopyFilter
+                ret = this._copyFilter(streamId, x, y, width, height,
+                    data, display, depth, data_index);
+                break;
+            case 1: // PaletteFilter
+                ret = this._paletteFilter(streamId, x, y, width, height,
+                    data, display, depth);
+                break;
+            case 2: // GradientFilter
+                ret = this._gradientFilter(streamId, x, y, width, height,
+                    data, display, depth);
+                break;
+            default:
+                throw new Error("Illegal tight filter received (ctl: " +
+                    this._filter + ")");
+        }
+
+        return ret;
+    }
+
+    _copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) {
+        const uncompressedSize = width * height * 3;
+
+        if (uncompressedSize === 0) {
+            return true;
+        }
+
+        if (uncompressedSize < 12) {
+            data = data.slice(data_index, data_index + uncompressedSize);
+        } else {
+            data = this._readData(data, data_index);
+            if (data === null) {
+                return false;
+            }
+
+            this._zlibs[streamId].setInput(data);
+            data = this._zlibs[streamId].inflate(uncompressedSize);
+            this._zlibs[streamId].setInput(null);
+        }
+
+        let rgbx = new Uint8Array(width * height * 4);
+        for (let i = 0, j = 0; i < width * height * 4; i += 4, j += 3) {
+            rgbx[i] = data[j];
+            rgbx[i + 1] = data[j + 1];
+            rgbx[i + 2] = data[j + 2];
+            rgbx[i + 3] = 255;  // Alpha
+        }
+
+        display.blitImage(x, y, width, height, rgbx, 0, false);
+
+        return true;
+    }
+
+    _paletteFilter(streamId, x, y, width, height, data, display, depth) {
+        const numColors = data[14] + 1;
+        const paletteSize = numColors * 3;
+        let palette = data.slice(15, 15 + paletteSize);
+
+        const bpp = (numColors <= 2) ? 1 : 8;
+        const rowSize = Math.floor((width * bpp + 7) / 8);
+        const uncompressedSize = rowSize * height;
+        let data_i = 15 + paletteSize;
+
+        if (uncompressedSize === 0) {
+            return true;
+        }
+
+        if (uncompressedSize < 12) {
+            data = data.slice(data_i, data_i + uncompressedSize);
+        } else {
+            data = this._readData(data, data_i);
+            if (data === null) {
+                return false;
+            }
+
+            this._zlibs[streamId].setInput(data);
+            data = this._zlibs[streamId].inflate(uncompressedSize);
+            this._zlibs[streamId].setInput(null);
+        }
+
+        // Convert indexed (palette based) image data to RGB
+        if (numColors == 2) {
+            this._monoRect(x, y, width, height, data, palette, display);
+        } else {
+            this._paletteRect(x, y, width, height, data, palette, display);
+        }
+
+        return true;
+    }
+
+    _monoRect(x, y, width, height, data, palette, display) {
+        // Convert indexed (palette based) image data to RGB
+        // TODO: reduce number of calculations inside loop
+        const dest = this._getScratchBuffer(width * height * 4);
+        const w = Math.floor((width + 7) / 8);
+        const w1 = Math.floor(width / 8);
+
+        for (let y = 0; y < height; y++) {
+            let dp, sp, x;
+            for (x = 0; x < w1; x++) {
+                for (let b = 7; b >= 0; b--) {
+                    dp = (y * width + x * 8 + 7 - b) * 4;
+                    sp = (data[y * w + x] >> b & 1) * 3;
+                    dest[dp] = palette[sp];
+                    dest[dp + 1] = palette[sp + 1];
+                    dest[dp + 2] = palette[sp + 2];
+                    dest[dp + 3] = 255;
+                }
+            }
+
+            for (let b = 7; b >= 8 - width % 8; b--) {
+                dp = (y * width + x * 8 + 7 - b) * 4;
+                sp = (data[y * w + x] >> b & 1) * 3;
+                dest[dp] = palette[sp];
+                dest[dp + 1] = palette[sp + 1];
+                dest[dp + 2] = palette[sp + 2];
+                dest[dp + 3] = 255;
+            }
+        }
+
+        display.blitImage(x, y, width, height, dest, 0, false);
+    }
+
+    _paletteRect(x, y, width, height, data, palette, display) {
+        // Convert indexed (palette based) image data to RGB
+        const dest = this._getScratchBuffer(width * height * 4);
+        const total = width * height * 4;
+        for (let i = 0, j = 0; i < total; i += 4, j++) {
+            const sp = data[j] * 3;
+            dest[i] = palette[sp];
+            dest[i + 1] = palette[sp + 1];
+            dest[i + 2] = palette[sp + 2];
+            dest[i + 3] = 255;
+        }
+
+        display.blitImage(x, y, width, height, dest, 0, false);
+    }
+
+    _gradientFilter(streamId, x, y, width, height, data, display, depth) {
+        throw new Error("Gradient filter not implemented");
+    }
+
+    _readData(data, len_index = 13) {
+        if (data.length < len_index + 2) {
+            Log.Error("UDP Decoder, readData, invalid data len")
+            return null;
+        }
+
+
+        let i = len_index;
+        let byte = data[i++];
+        let len = byte & 0x7f;
+        // lenth field is variably sized 1 to 3 bytes long
+        if (byte & 0x80) {
+            byte = data[i++]
+            len |= (byte & 0x7f) << 7;
+            if (byte & 0x80) {
+                byte = data[i++];
+                len |= byte << 14;
+            }
+        }
+
+        //TODO: get rid of me
+        if (data.length !== len + i) {
+            console.log('Rect of size ' + len + ' with data size ' + data.length + ' index of ' + i);
+        }
+        
+
+        return data.slice(i);
+    }
+
+    _getScratchBuffer(size) {
+        if (!this._scratchBuffer || (this._scratchBuffer.length < size)) {
+            this._scratchBuffer = new Uint8Array(size);
+        }
+        return this._scratchBuffer;
+    }
+}

core/display.js

@@ -329,11 +329,9 @@ export default class Display {
                                           x, y, w, h,
                                           vx, vy, w, h);
 
-                if (this.isNewFrame(x, y, h, w)) {
+                this._flipCnt += 1;
-                    this._flipCnt += 1;
-                }
             }
-
+	  
             this._damageBounds.left = this._damageBounds.top = 65535;
             this._damageBounds.right = this._damageBounds.bottom = 0;
         }

core/encodings.js

@@ -13,6 +13,7 @@ export const encodings = {
     encodingHextile: 5,
     encodingTight: 7,
     encodingTightPNG: -260,
+    encodingUDP: -261,
 
     pseudoEncodingQualityLevel9: -23,
     pseudoEncodingQualityLevel0: -32,

core/rfb.js

@@ -33,6 +33,7 @@ import RREDecoder from "./decoders/rre.js";
 import HextileDecoder from "./decoders/hextile.js";
 import TightDecoder from "./decoders/tight.js";
 import TightPNGDecoder from "./decoders/tightpng.js";
+import UDPDecoder from './decoders/udp.js';
 import { toSignedRelative16bit } from './util/int.js';
 
 // How many seconds to wait for a disconnect to finish
@@ -136,6 +137,7 @@ export default class RFB extends EventTargetMixin {
         this._maxVideoResolutionX = 960;
         this._maxVideoResolutionY = 540;
         this._clipboardBinary = true;
+        this._useUdp = true;
 
         this._trackFrameStats = false;
 
@@ -239,6 +241,7 @@ export default class RFB extends EventTargetMixin {
         this._decoders[encodings.encodingHextile] = new HextileDecoder();
         this._decoders[encodings.encodingTight] = new TightDecoder();
         this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder();
+        this._decoders[encodings.encodingUDP] = new UDPDecoder();
 
         // NB: nothing that needs explicit teardown should be done
         // before this point, since this can throw an exception
@@ -973,6 +976,105 @@ export default class RFB extends EventTargetMixin {
         this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
         this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
 
+        // WebRTC UDP datachannel inits
+        {
+            this._udpBuffer = new Map();
+
+            this._udpPeer = new RTCPeerConnection({
+                iceServers: [{
+                    urls: ["stun:stun.l.google.com:19302"]
+                }]
+            });
+            let peer = this._udpPeer;
+
+            peer.onicecandidate = function(e) {
+                if (e.candidate)
+                    Log.Debug("received ice candidate", e.candidate);
+                else
+                    Log.Debug("all candidates received");
+            }
+
+            peer.ondatachannel = function(e) {
+                Log.Debug("peer connection on data channel", e);
+            }
+
+            this._udpChannel = peer.createDataChannel("webudp", {
+                ordered: false,
+                maxRetransmits: 0
+            });
+            this._udpChannel.binaryType = "arraybuffer";
+
+            this._udpChannel.onerror = function(e) {
+                Log.Error("data channel error " + e.message);
+            }
+
+            let sock = this._sock;
+            let udpBuffer = this._udpBuffer;
+            let me = this;
+            this._udpChannel.onmessage = function(e) {
+                //Log.Info("got udp msg", e.data);
+                const u8 = new Uint8Array(e.data);
+                // Got an UDP packet. Do we need reassembly?
+                const id = parseInt(u8[0] +
+                                    (u8[1] << 8) +
+                                    (u8[2] << 16) +
+                                    (u8[3] << 24), 10);
+                const i = parseInt(u8[4] +
+                                   (u8[5] << 8) +
+                                   (u8[6] << 16) +
+                                   (u8[7] << 24), 10);
+                const pieces = parseInt(u8[8] +
+                                        (u8[9] << 8) +
+                                        (u8[10] << 16) +
+                                        (u8[11] << 24), 10);
+                const hash = parseInt(u8[12] +
+                                        (u8[13] << 8) +
+                                        (u8[14] << 16) +
+                                        (u8[15] << 24), 10);
+                // TODO: check the hash. It's the low 32 bits of XXH64, seed 0
+
+                if (pieces == 1) { // Handle it immediately
+                    me._handleUdpRect(u8.slice(16));
+                } else { // Insert into wait array
+                    const now = Date.now();
+		    
+                    if (udpBuffer.has(id)) {
+                        let item = udpBuffer.get(id);
+                        item.recieved_pieces += 1;
+                        item.data[i] = u8.slice(16);
+                        item.total_bytes += item.data[i].length;
+
+                        if (item.total_pieces == item.recieved_pieces) {
+                            // Message is complete, combile data into a single array
+                            var finaldata = new Uint8Array(item.total_bytes);
+                            let z = 0;
+                            for (let x = 0; x < item.data.length; x++) {
+                                finaldata.set(item.data[x], z);
+                                z += item.data[x].length;
+                            }
+                            udpBuffer.delete(id);
+                            me._handleUdpRect(finaldata);
+                        }
+                    } else {
+                        let item = {
+                            total_pieces: pieces,   // number of pieces expected
+                                arrival: now,       //time first piece was recieved
+                            recieved_pieces: 1,     // current number of pieces in data
+                            total_bytes: 0,         // total size of all data pieces combined
+                            data: new Array(pieces)
+                        }
+                        item.data[i] = u8.slice(16);
+                        item.total_bytes = item.data[i].length;
+                        udpBuffer.set(id, item);
+                    }
+                }
+            }
+        }
+
+	    if (this._useUdp) {
+            setTimeout(function() { this._sendUdpUpgrade() }.bind(this), 3000);
+        }
+
         Log.Debug("<< RFB.connect");
     }
 
@@ -2853,6 +2955,9 @@ export default class RFB extends EventTargetMixin {
             case 180: // KASM binary clipboard
                 return this._handleBinaryClipboard();
 
+            case 181: // KASM UDP upgrade
+                return this._handleUdpUpgrade();
+
             case 248: // ServerFence
                 return this._handleServerFenceMsg();
 
@@ -2874,6 +2979,101 @@ export default class RFB extends EventTargetMixin {
         }
     }
 
+    _handleUdpRect(data) {
+        let frame = {
+            x: (data[0] << 8) + data[1],
+            y: (data[2] << 8) + data[3],
+            width: (data[4] << 8) + data[5],
+            height: (data[6] << 8) + data[7],
+            encoding: parseInt((data[8] << 24) + (data[9] << 16) +
+                                            (data[10] << 8) + data[11], 10)
+        };
+
+        switch (frame.encoding) {
+            case encodings.pseudoEncodingLastRect:
+                if (document.visibilityState !== "hidden") {
+                    this._display.flip();
+                    this._udpBuffer.clear();
+                }
+                break;
+            case encodings.encodingTight:
+                let decoder = this._decoders[encodings.encodingUDP];
+                try {
+                    decoder.decodeRect(frame.x, frame.y,
+                        frame.width, frame.height,
+                        data, this._display,
+                        this._fbDepth);
+                } catch (err) {
+                    this._fail("Error decoding rect: " + err);
+                    return false;
+                }
+                break;
+            default:
+                Log.Error("Invalid rect encoding via UDP: " + frame.encoding);
+                return false;
+        }
+
+        return true;
+    }
+
+    _sendUdpUpgrade() {
+        let peer = this._udpPeer;
+        let sock = this._sock;
+
+        peer.createOffer().then(function(offer) {
+            return peer.setLocalDescription(offer);
+        }).then(function() {
+            const buff = sock._sQ;
+            const offset = sock._sQlen;
+            const str = Uint8Array.from(Array.from(peer.localDescription.sdp).map(letter => letter.charCodeAt(0)));
+
+            buff[offset] = 181; // msg-type
+            buff[offset + 1] = str.length >> 8; // u16 len
+            buff[offset + 2] = str.length;
+
+            buff.set(str, offset + 3);
+
+            sock._sQlen += 3 + str.length;
+            sock.flush();
+        }).catch(function(reason) {
+            Log.Error("Failed to create offer " + reason);
+        });
+    }
+
+    _sendUdpDowngrade() {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        buff[offset] = 181; // msg-type
+        buff[offset + 1] = 0; // u16 len
+        buff[offset + 2] = 0;
+
+        sock._sQlen += 3;
+        sock.flush();
+    }
+
+    _handleUdpUpgrade() {
+        if (this._sock.rQwait("UdpUgrade header", 2, 1)) { return false; }
+        let len = this._sock.rQshift16();
+        if (this._sock.rQwait("UdpUpgrade payload", len, 3)) { return false; }
+
+        const payload = this._sock.rQshiftStr(len);
+
+        let peer = this._udpPeer;
+
+        var response = JSON.parse(payload);
+        peer.setRemoteDescription(new RTCSessionDescription(response.answer)).then(function() {
+            var candidate = new RTCIceCandidate(response.candidate);
+            peer.addIceCandidate(candidate).then(function() {
+                Log.Debug("success in addicecandidate");
+            }).catch(function(err) {
+                Log.Error("Failure in addIceCandidate", err);
+            });
+        }).catch(function(e) {
+            Log.Error("Failure in setRemoteDescription", e);
+        });
+    }
+
     _framebufferUpdate() {
         if (this._FBU.rects === 0) {
             if (this._sock.rQwait("FBU header", 3, 1)) { return false; }
@@ -3231,7 +3431,7 @@ export default class RFB extends EventTargetMixin {
                                       this._sock, this._display,
                                       this._fbDepth);
         } catch (err) {
-            this._fail("Error decoding rect: " + err);
+	    this._fail("Error decoding rect: " + err);
             return false;
         }
     }

core/websock.js

@@ -321,6 +321,19 @@ export default class Websock {
         this._rQlen += u8.length;
     }
 
+    // Insert some new data into the current position, pushing the old data back
+    _insertIntoMiddle(data) {
+        const u8 = new Uint8Array(data);
+        if (u8.length > this._rQbufferSize - this._rQlen) {
+            this._expandCompactRQ(u8.length);
+        }
+
+        this._rQ.copyWithin(this._rQi + u8.length, this._rQi, this._rQlen - this._rQi);
+
+        this._rQ.set(u8, this._rQi);
+        this._rQlen += u8.length;
+    }
+
     _recvMessage(e) {
         this._DecodeMessage(e.data);
         if (this.rQlen > 0) {