Udp Initial MVP (#32)
* Initial UDP code * refactored UDP reassembly * WIP Added udp decoder * WIP cleanup * Port multiplexing * UDP upgrade/downgrade * UDP packets now have a hash * Correct udp.js this.numColors bug * auto upgrade to udp * re-apply webpack * Code cleanup, optimize clearing udpBuffer Co-authored-by: Lauri Kasanen <cand@gmx.com> Co-authored-by: matt <matt@kasmweb.com>
This commit is contained in:
Matt McClaskey
GitHub
parent
7e5a302a7a
commit
5a8d8f24b4
|
|
@ -0,0 +1,288 @@
|
|||
/*
|
||||
* noVNC: HTML5 VNC client
|
||||
* Copyright (C) 2019 The noVNC Authors
|
||||
* (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca)
|
||||
* Licensed under MPL 2.0 (see LICENSE.txt)
|
||||
*
|
||||
* See README.md for usage and integration instructions.
|
||||
*
|
||||
*/
|
||||
|
||||
import * as Log from '../util/logging.js';
|
||||
import Inflator from "../inflator.js";
|
||||
|
||||
export default class UDPDecoder {
|
||||
constructor() {
|
||||
this._filter = null;
|
||||
this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel)
|
||||
|
||||
this._zlibs = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
this._zlibs[i] = new Inflator();
|
||||
}
|
||||
}
|
||||
|
||||
decodeRect(x, y, width, height, data, display, depth) {
|
||||
let ctl = data[12];
|
||||
ctl = ctl >> 4;
|
||||
|
||||
let ret;
|
||||
|
||||
if (ctl === 0x08) {
|
||||
ret = this._fillRect(x, y, width, height,
|
||||
data, display, depth);
|
||||
} else if (ctl === 0x09) {
|
||||
ret = this._jpegRect(x, y, width, height,
|
||||
data, display, depth);
|
||||
} else if (ctl === 0x0A) {
|
||||
ret = this._pngRect(x, y, width, height,
|
||||
data, display, depth);
|
||||
} else if ((ctl & 0x08) == 0) {
|
||||
ret = this._basicRect(ctl, x, y, width, height,
|
||||
data, display, depth);
|
||||
} else if (ctl === 0x0B) {
|
||||
ret = this._webpRect(x, y, width, height,
|
||||
data, display, depth);
|
||||
} else {
|
||||
throw new Error("Illegal udp compression received (ctl: " +
|
||||
ctl + ")");
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
_fillRect(x, y, width, height, data, display, depth) {
|
||||
|
||||
display.fillRect(x, y, width, height,
|
||||
[data[13], data[14], data[15]], false);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_jpegRect(x, y, width, height, data, display, depth) {
|
||||
let img = this._readData(data);
|
||||
if (img === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
display.imageRect(x, y, width, height, "image/jpeg", img);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_webpRect(x, y, width, height, data, display, depth) {
|
||||
let img = this._readData(data);
|
||||
if (img === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
display.imageRect(x, y, width, height, "image/webp", img);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_pngRect(x, y, width, height, data, display, depth) {
|
||||
//throw new Error("PNG received in UDP rect");
|
||||
Log.Error("PNG received in UDP rect");
|
||||
}
|
||||
|
||||
_basicRect(ctl, x, y, width, height, data, display, depth) {
|
||||
let zlibs_flags = data[12];
|
||||
// Reset streams if the server requests it
|
||||
for (let i = 0; i < 4; i++) {
|
||||
if ((zlibs_flags >> i) & 1) {
|
||||
this._zlibs[i].reset();
|
||||
Log.Info("Reset zlib stream " + i);
|
||||
}
|
||||
}
|
||||
|
||||
let filter = data[13];
|
||||
let data_index = 14;
|
||||
let streamId = ctl & 0x3;
|
||||
if (!(ctl & 0x4)) {
|
||||
// Implicit CopyFilter
|
||||
filter = 0;
|
||||
data_index = 13;
|
||||
}
|
||||
|
||||
let ret;
|
||||
|
||||
switch (filter) {
|
||||
case 0: // CopyFilter
|
||||
ret = this._copyFilter(streamId, x, y, width, height,
|
||||
data, display, depth, data_index);
|
||||
break;
|
||||
case 1: // PaletteFilter
|
||||
ret = this._paletteFilter(streamId, x, y, width, height,
|
||||
data, display, depth);
|
||||
break;
|
||||
case 2: // GradientFilter
|
||||
ret = this._gradientFilter(streamId, x, y, width, height,
|
||||
data, display, depth);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Illegal tight filter received (ctl: " +
|
||||
this._filter + ")");
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
_copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) {
|
||||
const uncompressedSize = width * height * 3;
|
||||
|
||||
if (uncompressedSize === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (uncompressedSize < 12) {
|
||||
data = data.slice(data_index, data_index + uncompressedSize);
|
||||
} else {
|
||||
data = this._readData(data, data_index);
|
||||
if (data === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this._zlibs[streamId].setInput(data);
|
||||
data = this._zlibs[streamId].inflate(uncompressedSize);
|
||||
this._zlibs[streamId].setInput(null);
|
||||
}
|
||||
|
||||
let rgbx = new Uint8Array(width * height * 4);
|
||||
for (let i = 0, j = 0; i < width * height * 4; i += 4, j += 3) {
|
||||
rgbx[i] = data[j];
|
||||
rgbx[i + 1] = data[j + 1];
|
||||
rgbx[i + 2] = data[j + 2];
|
||||
rgbx[i + 3] = 255; // Alpha
|
||||
}
|
||||
|
||||
display.blitImage(x, y, width, height, rgbx, 0, false);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_paletteFilter(streamId, x, y, width, height, data, display, depth) {
|
||||
const numColors = data[14] + 1;
|
||||
const paletteSize = numColors * 3;
|
||||
let palette = data.slice(15, 15 + paletteSize);
|
||||
|
||||
const bpp = (numColors <= 2) ? 1 : 8;
|
||||
const rowSize = Math.floor((width * bpp + 7) / 8);
|
||||
const uncompressedSize = rowSize * height;
|
||||
let data_i = 15 + paletteSize;
|
||||
|
||||
if (uncompressedSize === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (uncompressedSize < 12) {
|
||||
data = data.slice(data_i, data_i + uncompressedSize);
|
||||
} else {
|
||||
data = this._readData(data, data_i);
|
||||
if (data === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this._zlibs[streamId].setInput(data);
|
||||
data = this._zlibs[streamId].inflate(uncompressedSize);
|
||||
this._zlibs[streamId].setInput(null);
|
||||
}
|
||||
|
||||
// Convert indexed (palette based) image data to RGB
|
||||
if (numColors == 2) {
|
||||
this._monoRect(x, y, width, height, data, palette, display);
|
||||
} else {
|
||||
this._paletteRect(x, y, width, height, data, palette, display);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_monoRect(x, y, width, height, data, palette, display) {
|
||||
// Convert indexed (palette based) image data to RGB
|
||||
// TODO: reduce number of calculations inside loop
|
||||
const dest = this._getScratchBuffer(width * height * 4);
|
||||
const w = Math.floor((width + 7) / 8);
|
||||
const w1 = Math.floor(width / 8);
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
let dp, sp, x;
|
||||
for (x = 0; x < w1; x++) {
|
||||
for (let b = 7; b >= 0; b--) {
|
||||
dp = (y * width + x * 8 + 7 - b) * 4;
|
||||
sp = (data[y * w + x] >> b & 1) * 3;
|
||||
dest[dp] = palette[sp];
|
||||
dest[dp + 1] = palette[sp + 1];
|
||||
dest[dp + 2] = palette[sp + 2];
|
||||
dest[dp + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
for (let b = 7; b >= 8 - width % 8; b--) {
|
||||
dp = (y * width + x * 8 + 7 - b) * 4;
|
||||
sp = (data[y * w + x] >> b & 1) * 3;
|
||||
dest[dp] = palette[sp];
|
||||
dest[dp + 1] = palette[sp + 1];
|
||||
dest[dp + 2] = palette[sp + 2];
|
||||
dest[dp + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
display.blitImage(x, y, width, height, dest, 0, false);
|
||||
}
|
||||
|
||||
_paletteRect(x, y, width, height, data, palette, display) {
|
||||
// Convert indexed (palette based) image data to RGB
|
||||
const dest = this._getScratchBuffer(width * height * 4);
|
||||
const total = width * height * 4;
|
||||
for (let i = 0, j = 0; i < total; i += 4, j++) {
|
||||
const sp = data[j] * 3;
|
||||
dest[i] = palette[sp];
|
||||
dest[i + 1] = palette[sp + 1];
|
||||
dest[i + 2] = palette[sp + 2];
|
||||
dest[i + 3] = 255;
|
||||
}
|
||||
|
||||
display.blitImage(x, y, width, height, dest, 0, false);
|
||||
}
|
||||
|
||||
_gradientFilter(streamId, x, y, width, height, data, display, depth) {
|
||||
throw new Error("Gradient filter not implemented");
|
||||
}
|
||||
|
||||
_readData(data, len_index = 13) {
|
||||
if (data.length < len_index + 2) {
|
||||
Log.Error("UDP Decoder, readData, invalid data len")
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
let i = len_index;
|
||||
let byte = data[i++];
|
||||
let len = byte & 0x7f;
|
||||
// lenth field is variably sized 1 to 3 bytes long
|
||||
if (byte & 0x80) {
|
||||
byte = data[i++]
|
||||
len |= (byte & 0x7f) << 7;
|
||||
if (byte & 0x80) {
|
||||
byte = data[i++];
|
||||
len |= byte << 14;
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: get rid of me
|
||||
if (data.length !== len + i) {
|
||||
console.log('Rect of size ' + len + ' with data size ' + data.length + ' index of ' + i);
|
||||
}
|
||||
|
||||
|
||||
return data.slice(i);
|
||||
}
|
||||
|
||||
_getScratchBuffer(size) {
|
||||
if (!this._scratchBuffer || (this._scratchBuffer.length < size)) {
|
||||
this._scratchBuffer = new Uint8Array(size);
|
||||
}
|
||||
return this._scratchBuffer;
|
||||
}
|
||||
}
|
||||
|
|
@ -329,10 +329,8 @@ export default class Display {
|
|||
x, y, w, h,
|
||||
vx, vy, w, h);
|
||||
|
||||
if (this.isNewFrame(x, y, h, w)) {
|
||||
this._flipCnt += 1;
|
||||
}
|
||||
}
|
||||
|
||||
this._damageBounds.left = this._damageBounds.top = 65535;
|
||||
this._damageBounds.right = this._damageBounds.bottom = 0;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ export const encodings = {
|
|||
encodingHextile: 5,
|
||||
encodingTight: 7,
|
||||
encodingTightPNG: -260,
|
||||
encodingUDP: -261,
|
||||
|
||||
pseudoEncodingQualityLevel9: -23,
|
||||
pseudoEncodingQualityLevel0: -32,
|
||||
|
|
|
|||
200
core/rfb.js
200
core/rfb.js
|
|
@ -33,6 +33,7 @@ import RREDecoder from "./decoders/rre.js";
|
|||
import HextileDecoder from "./decoders/hextile.js";
|
||||
import TightDecoder from "./decoders/tight.js";
|
||||
import TightPNGDecoder from "./decoders/tightpng.js";
|
||||
import UDPDecoder from './decoders/udp.js';
|
||||
import { toSignedRelative16bit } from './util/int.js';
|
||||
|
||||
// How many seconds to wait for a disconnect to finish
|
||||
|
|
@ -136,6 +137,7 @@ export default class RFB extends EventTargetMixin {
|
|||
this._maxVideoResolutionX = 960;
|
||||
this._maxVideoResolutionY = 540;
|
||||
this._clipboardBinary = true;
|
||||
this._useUdp = true;
|
||||
|
||||
this._trackFrameStats = false;
|
||||
|
||||
|
|
@ -239,6 +241,7 @@ export default class RFB extends EventTargetMixin {
|
|||
this._decoders[encodings.encodingHextile] = new HextileDecoder();
|
||||
this._decoders[encodings.encodingTight] = new TightDecoder();
|
||||
this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder();
|
||||
this._decoders[encodings.encodingUDP] = new UDPDecoder();
|
||||
|
||||
// NB: nothing that needs explicit teardown should be done
|
||||
// before this point, since this can throw an exception
|
||||
|
|
@ -973,6 +976,105 @@ export default class RFB extends EventTargetMixin {
|
|||
this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
|
||||
this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
|
||||
|
||||
// WebRTC UDP datachannel inits
|
||||
{
|
||||
this._udpBuffer = new Map();
|
||||
|
||||
this._udpPeer = new RTCPeerConnection({
|
||||
iceServers: [{
|
||||
urls: ["stun:stun.l.google.com:19302"]
|
||||
}]
|
||||
});
|
||||
let peer = this._udpPeer;
|
||||
|
||||
peer.onicecandidate = function(e) {
|
||||
if (e.candidate)
|
||||
Log.Debug("received ice candidate", e.candidate);
|
||||
else
|
||||
Log.Debug("all candidates received");
|
||||
}
|
||||
|
||||
peer.ondatachannel = function(e) {
|
||||
Log.Debug("peer connection on data channel", e);
|
||||
}
|
||||
|
||||
this._udpChannel = peer.createDataChannel("webudp", {
|
||||
ordered: false,
|
||||
maxRetransmits: 0
|
||||
});
|
||||
this._udpChannel.binaryType = "arraybuffer";
|
||||
|
||||
this._udpChannel.onerror = function(e) {
|
||||
Log.Error("data channel error " + e.message);
|
||||
}
|
||||
|
||||
let sock = this._sock;
|
||||
let udpBuffer = this._udpBuffer;
|
||||
let me = this;
|
||||
this._udpChannel.onmessage = function(e) {
|
||||
//Log.Info("got udp msg", e.data);
|
||||
const u8 = new Uint8Array(e.data);
|
||||
// Got an UDP packet. Do we need reassembly?
|
||||
const id = parseInt(u8[0] +
|
||||
(u8[1] << 8) +
|
||||
(u8[2] << 16) +
|
||||
(u8[3] << 24), 10);
|
||||
const i = parseInt(u8[4] +
|
||||
(u8[5] << 8) +
|
||||
(u8[6] << 16) +
|
||||
(u8[7] << 24), 10);
|
||||
const pieces = parseInt(u8[8] +
|
||||
(u8[9] << 8) +
|
||||
(u8[10] << 16) +
|
||||
(u8[11] << 24), 10);
|
||||
const hash = parseInt(u8[12] +
|
||||
(u8[13] << 8) +
|
||||
(u8[14] << 16) +
|
||||
(u8[15] << 24), 10);
|
||||
// TODO: check the hash. It's the low 32 bits of XXH64, seed 0
|
||||
|
||||
if (pieces == 1) { // Handle it immediately
|
||||
me._handleUdpRect(u8.slice(16));
|
||||
} else { // Insert into wait array
|
||||
const now = Date.now();
|
||||
|
||||
if (udpBuffer.has(id)) {
|
||||
let item = udpBuffer.get(id);
|
||||
item.recieved_pieces += 1;
|
||||
item.data[i] = u8.slice(16);
|
||||
item.total_bytes += item.data[i].length;
|
||||
|
||||
if (item.total_pieces == item.recieved_pieces) {
|
||||
// Message is complete, combile data into a single array
|
||||
var finaldata = new Uint8Array(item.total_bytes);
|
||||
let z = 0;
|
||||
for (let x = 0; x < item.data.length; x++) {
|
||||
finaldata.set(item.data[x], z);
|
||||
z += item.data[x].length;
|
||||
}
|
||||
udpBuffer.delete(id);
|
||||
me._handleUdpRect(finaldata);
|
||||
}
|
||||
} else {
|
||||
let item = {
|
||||
total_pieces: pieces, // number of pieces expected
|
||||
arrival: now, //time first piece was recieved
|
||||
recieved_pieces: 1, // current number of pieces in data
|
||||
total_bytes: 0, // total size of all data pieces combined
|
||||
data: new Array(pieces)
|
||||
}
|
||||
item.data[i] = u8.slice(16);
|
||||
item.total_bytes = item.data[i].length;
|
||||
udpBuffer.set(id, item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (this._useUdp) {
|
||||
setTimeout(function() { this._sendUdpUpgrade() }.bind(this), 3000);
|
||||
}
|
||||
|
||||
Log.Debug("<< RFB.connect");
|
||||
}
|
||||
|
||||
|
|
@ -2853,6 +2955,9 @@ export default class RFB extends EventTargetMixin {
|
|||
case 180: // KASM binary clipboard
|
||||
return this._handleBinaryClipboard();
|
||||
|
||||
case 181: // KASM UDP upgrade
|
||||
return this._handleUdpUpgrade();
|
||||
|
||||
case 248: // ServerFence
|
||||
return this._handleServerFenceMsg();
|
||||
|
||||
|
|
@ -2874,6 +2979,101 @@ export default class RFB extends EventTargetMixin {
|
|||
}
|
||||
}
|
||||
|
||||
_handleUdpRect(data) {
|
||||
let frame = {
|
||||
x: (data[0] << 8) + data[1],
|
||||
y: (data[2] << 8) + data[3],
|
||||
width: (data[4] << 8) + data[5],
|
||||
height: (data[6] << 8) + data[7],
|
||||
encoding: parseInt((data[8] << 24) + (data[9] << 16) +
|
||||
(data[10] << 8) + data[11], 10)
|
||||
};
|
||||
|
||||
switch (frame.encoding) {
|
||||
case encodings.pseudoEncodingLastRect:
|
||||
if (document.visibilityState !== "hidden") {
|
||||
this._display.flip();
|
||||
this._udpBuffer.clear();
|
||||
}
|
||||
break;
|
||||
case encodings.encodingTight:
|
||||
let decoder = this._decoders[encodings.encodingUDP];
|
||||
try {
|
||||
decoder.decodeRect(frame.x, frame.y,
|
||||
frame.width, frame.height,
|
||||
data, this._display,
|
||||
this._fbDepth);
|
||||
} catch (err) {
|
||||
this._fail("Error decoding rect: " + err);
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
Log.Error("Invalid rect encoding via UDP: " + frame.encoding);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_sendUdpUpgrade() {
|
||||
let peer = this._udpPeer;
|
||||
let sock = this._sock;
|
||||
|
||||
peer.createOffer().then(function(offer) {
|
||||
return peer.setLocalDescription(offer);
|
||||
}).then(function() {
|
||||
const buff = sock._sQ;
|
||||
const offset = sock._sQlen;
|
||||
const str = Uint8Array.from(Array.from(peer.localDescription.sdp).map(letter => letter.charCodeAt(0)));
|
||||
|
||||
buff[offset] = 181; // msg-type
|
||||
buff[offset + 1] = str.length >> 8; // u16 len
|
||||
buff[offset + 2] = str.length;
|
||||
|
||||
buff.set(str, offset + 3);
|
||||
|
||||
sock._sQlen += 3 + str.length;
|
||||
sock.flush();
|
||||
}).catch(function(reason) {
|
||||
Log.Error("Failed to create offer " + reason);
|
||||
});
|
||||
}
|
||||
|
||||
_sendUdpDowngrade() {
|
||||
const buff = sock._sQ;
|
||||
const offset = sock._sQlen;
|
||||
|
||||
buff[offset] = 181; // msg-type
|
||||
buff[offset + 1] = 0; // u16 len
|
||||
buff[offset + 2] = 0;
|
||||
|
||||
sock._sQlen += 3;
|
||||
sock.flush();
|
||||
}
|
||||
|
||||
_handleUdpUpgrade() {
|
||||
if (this._sock.rQwait("UdpUgrade header", 2, 1)) { return false; }
|
||||
let len = this._sock.rQshift16();
|
||||
if (this._sock.rQwait("UdpUpgrade payload", len, 3)) { return false; }
|
||||
|
||||
const payload = this._sock.rQshiftStr(len);
|
||||
|
||||
let peer = this._udpPeer;
|
||||
|
||||
var response = JSON.parse(payload);
|
||||
peer.setRemoteDescription(new RTCSessionDescription(response.answer)).then(function() {
|
||||
var candidate = new RTCIceCandidate(response.candidate);
|
||||
peer.addIceCandidate(candidate).then(function() {
|
||||
Log.Debug("success in addicecandidate");
|
||||
}).catch(function(err) {
|
||||
Log.Error("Failure in addIceCandidate", err);
|
||||
});
|
||||
}).catch(function(e) {
|
||||
Log.Error("Failure in setRemoteDescription", e);
|
||||
});
|
||||
}
|
||||
|
||||
_framebufferUpdate() {
|
||||
if (this._FBU.rects === 0) {
|
||||
if (this._sock.rQwait("FBU header", 3, 1)) { return false; }
|
||||
|
|
|
|||
|
|
@ -321,6 +321,19 @@ export default class Websock {
|
|||
this._rQlen += u8.length;
|
||||
}
|
||||
|
||||
// Insert some new data into the current position, pushing the old data back
|
||||
_insertIntoMiddle(data) {
|
||||
const u8 = new Uint8Array(data);
|
||||
if (u8.length > this._rQbufferSize - this._rQlen) {
|
||||
this._expandCompactRQ(u8.length);
|
||||
}
|
||||
|
||||
this._rQ.copyWithin(this._rQi + u8.length, this._rQi, this._rQlen - this._rQi);
|
||||
|
||||
this._rQ.set(u8, this._rQi);
|
||||
this._rQlen += u8.length;
|
||||
}
|
||||
|
||||
_recvMessage(e) {
|
||||
this._DecodeMessage(e.data);
|
||||
if (this.rQlen > 0) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue