From 6a19390baafedbc2729597fe244010a2aaadc068 Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Sun, 7 Jun 2020 14:22:07 +0200 Subject: [PATCH] Switch to RGBx pixel format This is what the browser wants so it avoids having to spend time converting everything. Unfortunately it usually means the server instead needs to convert it for us, but we assume it has more power than we do. --- core/decoders/hextile.js | 21 +++++++++++++-------- core/decoders/raw.js | 10 ++++++++-- core/decoders/tight.js | 26 +++++++++++++------------- core/display.js | 30 +++++++++++++++--------------- core/rfb.js | 4 ++-- core/util/browser.js | 9 +++++++++ tests/test.copyrect.js | 2 +- tests/test.display.js | 21 +++++++-------------- tests/test.hextile.js | 20 ++++++++++---------- tests/test.raw.js | 22 +++++++++++----------- tests/test.rre.js | 8 ++++---- 11 files changed, 93 insertions(+), 80 deletions(-) diff --git a/core/decoders/hextile.js b/core/decoders/hextile.js index f12e7f6a..ac21eff0 100644 --- a/core/decoders/hextile.js +++ b/core/decoders/hextile.js @@ -88,6 +88,11 @@ export default class HextileDecoder { display.fillRect(tx, ty, tw, th, this._background); } } else if (subencoding & 0x01) { // Raw + let pixels = tw * th; + // Max sure the image is fully opaque + for (let i = 0;i < pixels;i++) { + rQ[rQi + i * 4 + 3] = 255; + } display.blitImage(tx, ty, tw, th, rQ, rQi); rQi += bytes - 1; } else { @@ -143,24 +148,24 @@ export default class HextileDecoder { this._tileW = width; this._tileH = height; - const red = color[2]; + const red = color[0]; const green = color[1]; - const blue = color[0]; + const blue = color[2]; const data = this._tileBuffer; for (let i = 0; i < width * height * 4; i += 4) { - data[i] = blue; + data[i] = red; data[i + 1] = green; - data[i + 2] = red; + data[i + 2] = blue; data[i + 3] = 255; } } // update sub-rectangle of the current tile _subTile(x, y, w, h, color) { - const red = color[2]; + const red = color[0]; const green = color[1]; - const blue = color[0]; + const blue = color[2]; const xend = x + w; const yend = y + h; @@ -169,9 +174,9 @@ export default class HextileDecoder { for (let j = y; j < yend; j++) { for (let i = x; i < xend; i++) { const p = (i + (j * width)) * 4; - data[p] = blue; + data[p] = red; data[p + 1] = green; - data[p + 2] = red; + data[p + 2] = blue; data[p + 3] = 255; } } diff --git a/core/decoders/raw.js b/core/decoders/raw.js index 4d84d7d1..d7a77ec9 100644 --- a/core/decoders/raw.js +++ b/core/decoders/raw.js @@ -27,23 +27,29 @@ export default class RawDecoder { const curY = y + (height - this._lines); const currHeight = Math.min(this._lines, Math.floor(sock.rQlen / bytesPerLine)); + const pixels = width * currHeight; + let data = sock.rQ; let index = sock.rQi; // Convert data if needed if (depth == 8) { - const pixels = width * currHeight; const newdata = new Uint8Array(pixels * 4); for (let i = 0; i < pixels; i++) { newdata[i * 4 + 0] = ((data[index + i] >> 0) & 0x3) * 255 / 3; newdata[i * 4 + 1] = ((data[index + i] >> 2) & 0x3) * 255 / 3; newdata[i * 4 + 2] = ((data[index + i] >> 4) & 0x3) * 255 / 3; - newdata[i * 4 + 4] = 0; + newdata[i * 4 + 3] = 255; } data = newdata; index = 0; } + // Max sure the image is fully opaque + for (let i = 0; i < pixels; i++) { + data[i * 4 + 3] = 255; + } + display.blitImage(x, curY, width, currHeight, data, index); sock.rQskipBytes(currHeight * bytesPerLine); this._lines -= currHeight; diff --git a/core/decoders/tight.js b/core/decoders/tight.js index e61f1329..8bc97d10 100644 --- a/core/decoders/tight.js +++ b/core/decoders/tight.js @@ -80,7 +80,7 @@ export default class TightDecoder { const rQ = sock.rQ; display.fillRect(x, y, width, height, - [rQ[rQi + 2], rQ[rQi + 1], rQ[rQi]], false); + [rQ[rQi], rQ[rQi + 1], rQ[rQi + 2]], false); sock.rQskipBytes(3); return true; @@ -165,15 +165,15 @@ export default class TightDecoder { this._zlibs[streamId].setInput(null); } - let bgrx = new Uint8Array(width * height * 4); + let rgbx = new Uint8Array(width * height * 4); for (let i = 0, j = 0; i < width * height * 4; i += 4, j += 3) { - bgrx[i] = data[j + 2]; - bgrx[i + 1] = data[j + 1]; - bgrx[i + 2] = data[j]; - bgrx[i + 3] = 255; // Alpha + rgbx[i] = data[j]; + rgbx[i + 1] = data[j + 1]; + rgbx[i + 2] = data[j + 2]; + rgbx[i + 3] = 255; // Alpha } - display.blitImage(x, y, width, height, bgrx, 0, false); + display.blitImage(x, y, width, height, rgbx, 0, false); return true; } @@ -245,9 +245,9 @@ export default class TightDecoder { for (let b = 7; b >= 0; b--) { dp = (y * width + x * 8 + 7 - b) * 4; sp = (data[y * w + x] >> b & 1) * 3; - dest[dp] = palette[sp + 2]; + dest[dp] = palette[sp]; dest[dp + 1] = palette[sp + 1]; - dest[dp + 2] = palette[sp]; + dest[dp + 2] = palette[sp + 2]; dest[dp + 3] = 255; } } @@ -255,9 +255,9 @@ export default class TightDecoder { for (let b = 7; b >= 8 - width % 8; b--) { dp = (y * width + x * 8 + 7 - b) * 4; sp = (data[y * w + x] >> b & 1) * 3; - dest[dp] = palette[sp + 2]; + dest[dp] = palette[sp]; dest[dp + 1] = palette[sp + 1]; - dest[dp + 2] = palette[sp]; + dest[dp + 2] = palette[sp + 2]; dest[dp + 3] = 255; } } @@ -271,9 +271,9 @@ export default class TightDecoder { const total = width * height * 4; for (let i = 0, j = 0; i < total; i += 4, j++) { const sp = data[j] * 3; - dest[i] = palette[sp + 2]; + dest[i] = palette[sp]; dest[i + 1] = palette[sp + 1]; - dest[i + 2] = palette[sp]; + dest[i + 2] = palette[sp + 2]; dest[i + 3] = 255; } diff --git a/core/display.js b/core/display.js index f06aa37c..494b7a37 100644 --- a/core/display.js +++ b/core/display.js @@ -8,6 +8,7 @@ import * as Log from './util/logging.js'; import Base64 from "./base64.js"; +import { supportsImageMetadata } from './util/browser.js'; export default class Display { constructor(target) { @@ -387,7 +388,19 @@ export default class Display { 'height': height, }); } else { - this._bgrxImageData(x, y, width, height, arr, offset); + // NB(directxman12): arr must be an Type Array view + let data = new Uint8ClampedArray(arr.buffer, + arr.byteOffset + offset, + width * height * 4); + let img; + if (supportsImageMetadata) { + img = new ImageData(data, width, height); + } else { + img = this._drawCtx.createImageData(width, height); + img.data.set(data); + } + this._drawCtx.putImageData(img, x, y); + this._damage(x, y, width, height); } } @@ -439,26 +452,13 @@ export default class Display { } _setFillColor(color) { - const newStyle = 'rgb(' + color[2] + ',' + color[1] + ',' + color[0] + ')'; + const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')'; if (newStyle !== this._prevDrawStyle) { this._drawCtx.fillStyle = newStyle; this._prevDrawStyle = newStyle; } } - _bgrxImageData(x, y, width, height, arr, offset) { - const img = this._drawCtx.createImageData(width, height); - const data = img.data; - for (let i = 0, j = offset; i < width * height * 4; i += 4, j += 4) { - data[i] = arr[j + 2]; - data[i + 1] = arr[j + 1]; - data[i + 2] = arr[j]; - data[i + 3] = 255; // Alpha - } - this._drawCtx.putImageData(img, x, y); - this._damage(x, y, img.width, img.height); - } - _renderQPush(action) { this._renderQ.push(action); if (this._renderQ.length === 1) { diff --git a/core/rfb.js b/core/rfb.js index ca50779e..40611966 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -2584,9 +2584,9 @@ RFB.messages = { buff[offset + 12] = 0; // blue-max buff[offset + 13] = (1 << bits) - 1; // blue-max - buff[offset + 14] = bits * 2; // red-shift + buff[offset + 14] = bits * 0; // red-shift buff[offset + 15] = bits * 1; // green-shift - buff[offset + 16] = bits * 0; // blue-shift + buff[offset + 16] = bits * 2; // blue-shift buff[offset + 17] = 0; // padding buff[offset + 18] = 0; // padding diff --git a/core/util/browser.js b/core/util/browser.js index a18d5a46..15548014 100644 --- a/core/util/browser.js +++ b/core/util/browser.js @@ -45,6 +45,15 @@ try { export const supportsCursorURIs = _supportsCursorURIs; +let _supportsImageMetadata = false; +try { + new ImageData(new Uint8ClampedArray(4), 1, 1); + _supportsImageMetadata = true; +} catch (ex) { + // ignore failure +} +export const supportsImageMetadata = _supportsImageMetadata; + let _hasScrollbarGutter = true; try { // Create invisible container diff --git a/tests/test.copyrect.js b/tests/test.copyrect.js index fcf42195..e2277cb5 100644 --- a/tests/test.copyrect.js +++ b/tests/test.copyrect.js @@ -38,7 +38,7 @@ describe('CopyRect Decoder', function () { it('should handle the CopyRect encoding', function () { // seed some initial data to copy display.fillRect(0, 0, 4, 4, [ 0x11, 0x22, 0x33 ]); - display.fillRect(0, 0, 2, 2, [ 0xff, 0x00, 0x00 ]); + display.fillRect(0, 0, 2, 2, [ 0x00, 0x00, 0xff ]); display.fillRect(2, 0, 2, 2, [ 0x00, 0xff, 0x00 ]); testDecodeRect(decoder, 0, 2, 2, 2, diff --git a/tests/test.display.js b/tests/test.display.js index 88c76074..bd5a55f9 100644 --- a/tests/test.display.js +++ b/tests/test.display.js @@ -128,7 +128,7 @@ describe('Display/Canvas Helper', function () { }); it('should keep the framebuffer data', function () { - display.fillRect(0, 0, 4, 4, [0, 0, 0xff]); + display.fillRect(0, 0, 4, 4, [0xff, 0, 0]); display.resize(2, 2); display.flip(); const expected = []; @@ -271,7 +271,7 @@ describe('Display/Canvas Helper', function () { }); it('should not draw directly on the target canvas', function () { - display.fillRect(0, 0, 4, 4, [0, 0, 0xff]); + display.fillRect(0, 0, 4, 4, [0xff, 0, 0]); display.flip(); display.fillRect(0, 0, 4, 4, [0, 0xff, 0]); const expected = []; @@ -285,15 +285,15 @@ describe('Display/Canvas Helper', function () { it('should support filling a rectangle with particular color via #fillRect', function () { display.fillRect(0, 0, 4, 4, [0, 0xff, 0]); - display.fillRect(0, 0, 2, 2, [0xff, 0, 0]); - display.fillRect(2, 2, 2, 2, [0xff, 0, 0]); + display.fillRect(0, 0, 2, 2, [0, 0, 0xff]); + display.fillRect(2, 2, 2, 2, [0, 0, 0xff]); display.flip(); expect(display).to.have.displayed(checkedData); }); it('should support copying an portion of the canvas via #copyImage', function () { display.fillRect(0, 0, 4, 4, [0, 0xff, 0]); - display.fillRect(0, 0, 2, 2, [0xff, 0, 0x00]); + display.fillRect(0, 0, 2, 2, [0, 0, 0xff]); display.copyImage(0, 0, 2, 2, 2, 2); display.flip(); expect(display).to.have.displayed(checkedData); @@ -309,15 +309,8 @@ describe('Display/Canvas Helper', function () { display.flush(); }); - it('should support drawing BGRX blit images with true color via #blitImage', function () { - const data = []; - for (let i = 0; i < 16; i++) { - data[i * 4] = checkedData[i * 4 + 2]; - data[i * 4 + 1] = checkedData[i * 4 + 1]; - data[i * 4 + 2] = checkedData[i * 4]; - data[i * 4 + 3] = checkedData[i * 4 + 3]; - } - display.blitImage(0, 0, 4, 4, data, 0); + it('should support blit images with true color via #blitImage', function () { + display.blitImage(0, 0, 4, 4, checkedData, 0); display.flip(); expect(display).to.have.displayed(checkedData); }); diff --git a/tests/test.hextile.js b/tests/test.hextile.js index 7c97527b..052c82e1 100644 --- a/tests/test.hextile.js +++ b/tests/test.hextile.js @@ -46,9 +46,9 @@ describe('Hextile Decoder', function () { let data = []; data.push(0x02 | 0x04 | 0x08); // bg spec, fg spec, anysubrects push32(data, 0x00ff0000); // becomes 00ff0000 --> #00FF00 bg color - data.push(0xff); // becomes ff000000 --> #0000FF fg color - data.push(0x00); + data.push(0x00); // becomes 0000ff00 --> #0000FF fg color data.push(0x00); + data.push(0xff); data.push(0x00); data.push(2); // 2 subrects data.push(0); // x: 0, y: 0 @@ -79,9 +79,9 @@ describe('Hextile Decoder', function () { let data = []; data.push(0x01); // raw for (let i = 0; i < targetData.length; i += 4) { - data.push(targetData[i + 2]); - data.push(targetData[i + 1]); data.push(targetData[i]); + data.push(targetData[i + 1]); + data.push(targetData[i + 2]); // Last byte zero to test correct alpha handling data.push(0); } @@ -137,15 +137,15 @@ describe('Hextile Decoder', function () { data.push(0x02 | 0x08 | 0x10); // bg spec, anysubrects, colouredsubrects push32(data, 0x00ff0000); // becomes 00ff0000 --> #00FF00 bg color data.push(2); // 2 subrects - data.push(0xff); // becomes ff000000 --> #0000FF fg color - data.push(0x00); + data.push(0x00); // becomes 0000ff00 --> #0000FF fg color data.push(0x00); + data.push(0xff); data.push(0x00); data.push(0); // x: 0, y: 0 data.push(1 | (1 << 4)); // width: 2, height: 2 - data.push(0xff); // becomes ff000000 --> #0000FF fg color - data.push(0x00); + data.push(0x00); // becomes 0000ff00 --> #0000FF fg color data.push(0x00); + data.push(0xff); data.push(0x00); data.push(2 | (2 << 4)); // x: 2, y: 2 data.push(1 | (1 << 4)); // width: 2, height: 2 @@ -168,10 +168,10 @@ describe('Hextile Decoder', function () { let data = []; data.push(0x02 | 0x04 | 0x08); // bg spec, fg spec, anysubrects push32(data, 0xff00ff); // becomes 00ff00ff --> #00FF00 bg color - data.push(0xff); // becomes ff0000ff --> #0000FF fg color - data.push(0x00); + data.push(0x00); // becomes 0000ffff --> #0000FF fg color data.push(0x00); data.push(0xff); + data.push(0xff); data.push(8); // 8 subrects for (let i = 0; i < 4; i++) { data.push((0 << 4) | (i * 4)); // x: 0, y: i*4 diff --git a/tests/test.raw.js b/tests/test.raw.js index b42e4f39..95a33af9 100644 --- a/tests/test.raw.js +++ b/tests/test.raw.js @@ -37,20 +37,20 @@ describe('Raw Decoder', function () { it('should handle the Raw encoding', function () { testDecodeRect(decoder, 0, 0, 2, 2, - [0x00, 0x00, 0xff, 0, 0x00, 0xff, 0x00, 0, - 0x00, 0xff, 0x00, 0, 0x00, 0x00, 0xff, 0], + [0xff, 0x00, 0x00, 0, 0x00, 0xff, 0x00, 0, + 0x00, 0xff, 0x00, 0, 0xff, 0x00, 0x00, 0], display, 24); testDecodeRect(decoder, 2, 0, 2, 2, - [0xff, 0x00, 0x00, 0, 0xff, 0x00, 0x00, 0, - 0xff, 0x00, 0x00, 0, 0xff, 0x00, 0x00, 0], + [0x00, 0x00, 0xff, 0, 0x00, 0x00, 0xff, 0, + 0x00, 0x00, 0xff, 0, 0x00, 0x00, 0xff, 0], display, 24); testDecodeRect(decoder, 0, 2, 4, 1, - [0xff, 0x00, 0xee, 0, 0xff, 0xee, 0x00, 0, - 0xff, 0xee, 0xaa, 0, 0xff, 0xee, 0xab, 0], + [0xee, 0x00, 0xff, 0, 0x00, 0xee, 0xff, 0, + 0xaa, 0xee, 0xff, 0, 0xab, 0xee, 0xff, 0], display, 24); testDecodeRect(decoder, 0, 3, 4, 1, - [0xff, 0x00, 0xee, 0, 0xff, 0xee, 0x00, 0, - 0xff, 0xee, 0xaa, 0, 0xff, 0xee, 0xab, 0], + [0xee, 0x00, 0xff, 0, 0x00, 0xee, 0xff, 0, + 0xaa, 0xee, 0xff, 0, 0xab, 0xee, 0xff, 0], display, 24); let targetData = new Uint8Array([ @@ -65,16 +65,16 @@ describe('Raw Decoder', function () { it('should handle the Raw encoding in low colour mode', function () { testDecodeRect(decoder, 0, 0, 2, 2, - [0x03, 0x03, 0x03, 0x03], + [0x30, 0x30, 0x30, 0x30], display, 8); testDecodeRect(decoder, 2, 0, 2, 2, [0x0c, 0x0c, 0x0c, 0x0c], display, 8); testDecodeRect(decoder, 0, 2, 4, 1, - [0x0c, 0x0c, 0x03, 0x03], + [0x0c, 0x0c, 0x30, 0x30], display, 8); testDecodeRect(decoder, 0, 3, 4, 1, - [0x0c, 0x0c, 0x03, 0x03], + [0x0c, 0x0c, 0x30, 0x30], display, 8); let targetData = new Uint8Array([ diff --git a/tests/test.rre.js b/tests/test.rre.js index 78b5eaae..fff1c945 100644 --- a/tests/test.rre.js +++ b/tests/test.rre.js @@ -53,17 +53,17 @@ describe('RRE Decoder', function () { let data = []; push32(data, 2); // 2 subrects push32(data, 0x00ff0000); // becomes 00ff0000 --> #00FF00 bg color - data.push(0xff); // becomes ff000000 --> #0000FF color - data.push(0x00); + data.push(0x00); // becomes 0000ff00 --> #0000FF fg color data.push(0x00); + data.push(0xff); data.push(0x00); push16(data, 0); // x: 0 push16(data, 0); // y: 0 push16(data, 2); // width: 2 push16(data, 2); // height: 2 - data.push(0xff); // becomes ff000000 --> #0000FF color - data.push(0x00); + data.push(0x00); // becomes 0000ff00 --> #0000FF fg color data.push(0x00); + data.push(0xff); data.push(0x00); push16(data, 2); // x: 2 push16(data, 2); // y: 2