diff --git a/core/decoders/udp.js b/core/decoders/udp.js new file mode 100644 index 00000000..a4516e91 --- /dev/null +++ b/core/decoders/udp.js @@ -0,0 +1,288 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2019 The noVNC Authors + * (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca) + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +import * as Log from '../util/logging.js'; +import Inflator from "../inflator.js"; + +export default class UDPDecoder { + constructor() { + this._filter = null; + this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel) + + this._zlibs = []; + for (let i = 0; i < 4; i++) { + this._zlibs[i] = new Inflator(); + } + } + + decodeRect(x, y, width, height, data, display, depth) { + let ctl = data[12]; + ctl = ctl >> 4; + + let ret; + + if (ctl === 0x08) { + ret = this._fillRect(x, y, width, height, + data, display, depth); + } else if (ctl === 0x09) { + ret = this._jpegRect(x, y, width, height, + data, display, depth); + } else if (ctl === 0x0A) { + ret = this._pngRect(x, y, width, height, + data, display, depth); + } else if ((ctl & 0x08) == 0) { + ret = this._basicRect(ctl, x, y, width, height, + data, display, depth); + } else if (ctl === 0x0B) { + ret = this._webpRect(x, y, width, height, + data, display, depth); + } else { + throw new Error("Illegal udp compression received (ctl: " + + ctl + ")"); + } + + return ret; + } + + _fillRect(x, y, width, height, data, display, depth) { + + display.fillRect(x, y, width, height, + [data[13], data[14], data[15]], false); + + return true; + } + + _jpegRect(x, y, width, height, data, display, depth) { + let img = this._readData(data); + if (img === null) { + return false; + } + + display.imageRect(x, y, width, height, "image/jpeg", img); + + return true; + } + + _webpRect(x, y, width, height, data, display, depth) { + let img = this._readData(data); + if (img === null) { + return false; + } + + display.imageRect(x, y, width, height, "image/webp", img); + + return true; + } + + _pngRect(x, y, width, height, data, display, depth) { + //throw new Error("PNG received in UDP rect"); + Log.Error("PNG received in UDP rect"); + } + + _basicRect(ctl, x, y, width, height, data, display, depth) { + let zlibs_flags = data[12]; + // Reset streams if the server requests it + for (let i = 0; i < 4; i++) { + if ((zlibs_flags >> i) & 1) { + this._zlibs[i].reset(); + Log.Info("Reset zlib stream " + i); + } + } + + let filter = data[13]; + let data_index = 14; + let streamId = ctl & 0x3; + if (!(ctl & 0x4)) { + // Implicit CopyFilter + filter = 0; + data_index = 13; + } + + let ret; + + switch (filter) { + case 0: // CopyFilter + ret = this._copyFilter(streamId, x, y, width, height, + data, display, depth, data_index); + break; + case 1: // PaletteFilter + ret = this._paletteFilter(streamId, x, y, width, height, + data, display, depth); + break; + case 2: // GradientFilter + ret = this._gradientFilter(streamId, x, y, width, height, + data, display, depth); + break; + default: + throw new Error("Illegal tight filter received (ctl: " + + this._filter + ")"); + } + + return ret; + } + + _copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) { + const uncompressedSize = width * height * 3; + + if (uncompressedSize === 0) { + return true; + } + + if (uncompressedSize < 12) { + data = data.slice(data_index, data_index + uncompressedSize); + } else { + data = this._readData(data, data_index); + if (data === null) { + return false; + } + + this._zlibs[streamId].setInput(data); + data = this._zlibs[streamId].inflate(uncompressedSize); + this._zlibs[streamId].setInput(null); + } + + let rgbx = new Uint8Array(width * height * 4); + for (let i = 0, j = 0; i < width * height * 4; i += 4, j += 3) { + rgbx[i] = data[j]; + rgbx[i + 1] = data[j + 1]; + rgbx[i + 2] = data[j + 2]; + rgbx[i + 3] = 255; // Alpha + } + + display.blitImage(x, y, width, height, rgbx, 0, false); + + return true; + } + + _paletteFilter(streamId, x, y, width, height, data, display, depth) { + const numColors = data[14] + 1; + const paletteSize = numColors * 3; + let palette = data.slice(15, 15 + paletteSize); + + const bpp = (numColors <= 2) ? 1 : 8; + const rowSize = Math.floor((width * bpp + 7) / 8); + const uncompressedSize = rowSize * height; + let data_i = 15 + paletteSize; + + if (uncompressedSize === 0) { + return true; + } + + if (uncompressedSize < 12) { + data = data.slice(data_i, data_i + uncompressedSize); + } else { + data = this._readData(data, data_i); + if (data === null) { + return false; + } + + this._zlibs[streamId].setInput(data); + data = this._zlibs[streamId].inflate(uncompressedSize); + this._zlibs[streamId].setInput(null); + } + + // Convert indexed (palette based) image data to RGB + if (numColors == 2) { + this._monoRect(x, y, width, height, data, palette, display); + } else { + this._paletteRect(x, y, width, height, data, palette, display); + } + + return true; + } + + _monoRect(x, y, width, height, data, palette, display) { + // Convert indexed (palette based) image data to RGB + // TODO: reduce number of calculations inside loop + const dest = this._getScratchBuffer(width * height * 4); + const w = Math.floor((width + 7) / 8); + const w1 = Math.floor(width / 8); + + for (let y = 0; y < height; y++) { + let dp, sp, x; + for (x = 0; x < w1; x++) { + for (let b = 7; b >= 0; b--) { + dp = (y * width + x * 8 + 7 - b) * 4; + sp = (data[y * w + x] >> b & 1) * 3; + dest[dp] = palette[sp]; + dest[dp + 1] = palette[sp + 1]; + dest[dp + 2] = palette[sp + 2]; + dest[dp + 3] = 255; + } + } + + for (let b = 7; b >= 8 - width % 8; b--) { + dp = (y * width + x * 8 + 7 - b) * 4; + sp = (data[y * w + x] >> b & 1) * 3; + dest[dp] = palette[sp]; + dest[dp + 1] = palette[sp + 1]; + dest[dp + 2] = palette[sp + 2]; + dest[dp + 3] = 255; + } + } + + display.blitImage(x, y, width, height, dest, 0, false); + } + + _paletteRect(x, y, width, height, data, palette, display) { + // Convert indexed (palette based) image data to RGB + const dest = this._getScratchBuffer(width * height * 4); + const total = width * height * 4; + for (let i = 0, j = 0; i < total; i += 4, j++) { + const sp = data[j] * 3; + dest[i] = palette[sp]; + dest[i + 1] = palette[sp + 1]; + dest[i + 2] = palette[sp + 2]; + dest[i + 3] = 255; + } + + display.blitImage(x, y, width, height, dest, 0, false); + } + + _gradientFilter(streamId, x, y, width, height, data, display, depth) { + throw new Error("Gradient filter not implemented"); + } + + _readData(data, len_index = 13) { + if (data.length < len_index + 2) { + Log.Error("UDP Decoder, readData, invalid data len") + return null; + } + + + let i = len_index; + let byte = data[i++]; + let len = byte & 0x7f; + // lenth field is variably sized 1 to 3 bytes long + if (byte & 0x80) { + byte = data[i++] + len |= (byte & 0x7f) << 7; + if (byte & 0x80) { + byte = data[i++]; + len |= byte << 14; + } + } + + //TODO: get rid of me + if (data.length !== len + i) { + console.log('Rect of size ' + len + ' with data size ' + data.length + ' index of ' + i); + } + + + return data.slice(i); + } + + _getScratchBuffer(size) { + if (!this._scratchBuffer || (this._scratchBuffer.length < size)) { + this._scratchBuffer = new Uint8Array(size); + } + return this._scratchBuffer; + } +} diff --git a/core/display.js b/core/display.js index ec11f0e8..ab31ba3b 100644 --- a/core/display.js +++ b/core/display.js @@ -329,11 +329,9 @@ export default class Display { x, y, w, h, vx, vy, w, h); - if (this.isNewFrame(x, y, h, w)) { - this._flipCnt += 1; - } + this._flipCnt += 1; } - + this._damageBounds.left = this._damageBounds.top = 65535; this._damageBounds.right = this._damageBounds.bottom = 0; } diff --git a/core/encodings.js b/core/encodings.js index d2f081ca..c89e3730 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -13,6 +13,7 @@ export const encodings = { encodingHextile: 5, encodingTight: 7, encodingTightPNG: -260, + encodingUDP: -261, pseudoEncodingQualityLevel9: -23, pseudoEncodingQualityLevel0: -32, diff --git a/core/rfb.js b/core/rfb.js index 74611321..4dccf3eb 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -33,6 +33,7 @@ import RREDecoder from "./decoders/rre.js"; import HextileDecoder from "./decoders/hextile.js"; import TightDecoder from "./decoders/tight.js"; import TightPNGDecoder from "./decoders/tightpng.js"; +import UDPDecoder from './decoders/udp.js'; import { toSignedRelative16bit } from './util/int.js'; // How many seconds to wait for a disconnect to finish @@ -136,6 +137,7 @@ export default class RFB extends EventTargetMixin { this._maxVideoResolutionX = 960; this._maxVideoResolutionY = 540; this._clipboardBinary = true; + this._useUdp = true; this._trackFrameStats = false; @@ -239,6 +241,7 @@ export default class RFB extends EventTargetMixin { this._decoders[encodings.encodingHextile] = new HextileDecoder(); this._decoders[encodings.encodingTight] = new TightDecoder(); this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); + this._decoders[encodings.encodingUDP] = new UDPDecoder(); // NB: nothing that needs explicit teardown should be done // before this point, since this can throw an exception @@ -973,6 +976,105 @@ export default class RFB extends EventTargetMixin { this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture); this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture); + // WebRTC UDP datachannel inits + { + this._udpBuffer = new Map(); + + this._udpPeer = new RTCPeerConnection({ + iceServers: [{ + urls: ["stun:stun.l.google.com:19302"] + }] + }); + let peer = this._udpPeer; + + peer.onicecandidate = function(e) { + if (e.candidate) + Log.Debug("received ice candidate", e.candidate); + else + Log.Debug("all candidates received"); + } + + peer.ondatachannel = function(e) { + Log.Debug("peer connection on data channel", e); + } + + this._udpChannel = peer.createDataChannel("webudp", { + ordered: false, + maxRetransmits: 0 + }); + this._udpChannel.binaryType = "arraybuffer"; + + this._udpChannel.onerror = function(e) { + Log.Error("data channel error " + e.message); + } + + let sock = this._sock; + let udpBuffer = this._udpBuffer; + let me = this; + this._udpChannel.onmessage = function(e) { + //Log.Info("got udp msg", e.data); + const u8 = new Uint8Array(e.data); + // Got an UDP packet. Do we need reassembly? + const id = parseInt(u8[0] + + (u8[1] << 8) + + (u8[2] << 16) + + (u8[3] << 24), 10); + const i = parseInt(u8[4] + + (u8[5] << 8) + + (u8[6] << 16) + + (u8[7] << 24), 10); + const pieces = parseInt(u8[8] + + (u8[9] << 8) + + (u8[10] << 16) + + (u8[11] << 24), 10); + const hash = parseInt(u8[12] + + (u8[13] << 8) + + (u8[14] << 16) + + (u8[15] << 24), 10); + // TODO: check the hash. It's the low 32 bits of XXH64, seed 0 + + if (pieces == 1) { // Handle it immediately + me._handleUdpRect(u8.slice(16)); + } else { // Insert into wait array + const now = Date.now(); + + if (udpBuffer.has(id)) { + let item = udpBuffer.get(id); + item.recieved_pieces += 1; + item.data[i] = u8.slice(16); + item.total_bytes += item.data[i].length; + + if (item.total_pieces == item.recieved_pieces) { + // Message is complete, combile data into a single array + var finaldata = new Uint8Array(item.total_bytes); + let z = 0; + for (let x = 0; x < item.data.length; x++) { + finaldata.set(item.data[x], z); + z += item.data[x].length; + } + udpBuffer.delete(id); + me._handleUdpRect(finaldata); + } + } else { + let item = { + total_pieces: pieces, // number of pieces expected + arrival: now, //time first piece was recieved + recieved_pieces: 1, // current number of pieces in data + total_bytes: 0, // total size of all data pieces combined + data: new Array(pieces) + } + item.data[i] = u8.slice(16); + item.total_bytes = item.data[i].length; + udpBuffer.set(id, item); + } + } + } + } + + if (this._useUdp) { + setTimeout(function() { this._sendUdpUpgrade() }.bind(this), 3000); + } + Log.Debug("<< RFB.connect"); } @@ -2853,6 +2955,9 @@ export default class RFB extends EventTargetMixin { case 180: // KASM binary clipboard return this._handleBinaryClipboard(); + case 181: // KASM UDP upgrade + return this._handleUdpUpgrade(); + case 248: // ServerFence return this._handleServerFenceMsg(); @@ -2874,6 +2979,101 @@ export default class RFB extends EventTargetMixin { } } + _handleUdpRect(data) { + let frame = { + x: (data[0] << 8) + data[1], + y: (data[2] << 8) + data[3], + width: (data[4] << 8) + data[5], + height: (data[6] << 8) + data[7], + encoding: parseInt((data[8] << 24) + (data[9] << 16) + + (data[10] << 8) + data[11], 10) + }; + + switch (frame.encoding) { + case encodings.pseudoEncodingLastRect: + if (document.visibilityState !== "hidden") { + this._display.flip(); + this._udpBuffer.clear(); + } + break; + case encodings.encodingTight: + let decoder = this._decoders[encodings.encodingUDP]; + try { + decoder.decodeRect(frame.x, frame.y, + frame.width, frame.height, + data, this._display, + this._fbDepth); + } catch (err) { + this._fail("Error decoding rect: " + err); + return false; + } + break; + default: + Log.Error("Invalid rect encoding via UDP: " + frame.encoding); + return false; + } + + return true; + } + + _sendUdpUpgrade() { + let peer = this._udpPeer; + let sock = this._sock; + + peer.createOffer().then(function(offer) { + return peer.setLocalDescription(offer); + }).then(function() { + const buff = sock._sQ; + const offset = sock._sQlen; + const str = Uint8Array.from(Array.from(peer.localDescription.sdp).map(letter => letter.charCodeAt(0))); + + buff[offset] = 181; // msg-type + buff[offset + 1] = str.length >> 8; // u16 len + buff[offset + 2] = str.length; + + buff.set(str, offset + 3); + + sock._sQlen += 3 + str.length; + sock.flush(); + }).catch(function(reason) { + Log.Error("Failed to create offer " + reason); + }); + } + + _sendUdpDowngrade() { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 181; // msg-type + buff[offset + 1] = 0; // u16 len + buff[offset + 2] = 0; + + sock._sQlen += 3; + sock.flush(); + } + + _handleUdpUpgrade() { + if (this._sock.rQwait("UdpUgrade header", 2, 1)) { return false; } + let len = this._sock.rQshift16(); + if (this._sock.rQwait("UdpUpgrade payload", len, 3)) { return false; } + + const payload = this._sock.rQshiftStr(len); + + let peer = this._udpPeer; + + var response = JSON.parse(payload); + peer.setRemoteDescription(new RTCSessionDescription(response.answer)).then(function() { + var candidate = new RTCIceCandidate(response.candidate); + peer.addIceCandidate(candidate).then(function() { + Log.Debug("success in addicecandidate"); + }).catch(function(err) { + Log.Error("Failure in addIceCandidate", err); + }); + }).catch(function(e) { + Log.Error("Failure in setRemoteDescription", e); + }); + } + _framebufferUpdate() { if (this._FBU.rects === 0) { if (this._sock.rQwait("FBU header", 3, 1)) { return false; } @@ -3231,7 +3431,7 @@ export default class RFB extends EventTargetMixin { this._sock, this._display, this._fbDepth); } catch (err) { - this._fail("Error decoding rect: " + err); + this._fail("Error decoding rect: " + err); return false; } } diff --git a/core/websock.js b/core/websock.js index 89ccdd94..32c6d7e8 100644 --- a/core/websock.js +++ b/core/websock.js @@ -321,6 +321,19 @@ export default class Websock { this._rQlen += u8.length; } + // Insert some new data into the current position, pushing the old data back + _insertIntoMiddle(data) { + const u8 = new Uint8Array(data); + if (u8.length > this._rQbufferSize - this._rQlen) { + this._expandCompactRQ(u8.length); + } + + this._rQ.copyWithin(this._rQi + u8.length, this._rQi, this._rQlen - this._rQi); + + this._rQ.set(u8, this._rQi); + this._rQlen += u8.length; + } + _recvMessage(e) { this._DecodeMessage(e.data); if (this.rQlen > 0) {