From e454666a8b2d741db746f96a7650fac6248a0471 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 2 Apr 2025 09:01:18 +0200 Subject: [PATCH 1/4] rfb: add qemu audio support Signed-off-by: Dietmar Maurer --- app/ui.js | 6 +++ core/encodings.js | 1 + core/rfb.js | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/app/ui.js b/app/ui.js index 51e57bd3..f45a1635 100644 --- a/app/ui.js +++ b/app/ui.js @@ -333,6 +333,12 @@ const UI = { .addEventListener('click', UI.rejectServer); document.getElementById("noVNC_credentials_button") .addEventListener('click', UI.setCredentials); + + document.addEventListener('click', function(event) { + if (UI.rfb !== undefined) { + UI.rfb.allow_audio(); + } + }); }, addClipboardHandlers() { diff --git a/core/encodings.js b/core/encodings.js index 7afcb17f..a69b4315 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -24,6 +24,7 @@ export const encodings = { pseudoEncodingLastRect: -224, pseudoEncodingCursor: -239, pseudoEncodingQEMUExtendedKeyEvent: -258, + pseudoEncodingQEMUAudioEvent: -259, pseudoEncodingQEMULedEvent: -261, pseudoEncodingDesktopName: -307, pseudoEncodingExtendedDesktopSize: -308, diff --git a/core/rfb.js b/core/rfb.js index e3266cc8..561bc4a7 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -154,6 +154,13 @@ export default class RFB extends EventTargetMixin { this._qemuExtKeyEventSupported = false; + this._qemuAudioSupported = false; + this._page_had_user_interaction = false; + this._audio_next_start = 0; + this._audio_sample_rate = 44100; + this._audio_channels = 2; + this._audio_context = null; + this._extendedPointerEventSupported = false; this._clipboardText = null; @@ -2256,6 +2263,7 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingDesktopSize); encs.push(encodings.pseudoEncodingLastRect); encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent); + encs.push(encodings.pseudoEncodingQEMUAudioEvent); encs.push(encodings.pseudoEncodingQEMULedEvent); encs.push(encodings.pseudoEncodingExtendedDesktopSize); encs.push(encodings.pseudoEncodingXvp); @@ -2611,6 +2619,9 @@ export default class RFB extends EventTargetMixin { case 250: // XVP return this._handleXvpMsg(); + case 255: // Qemu Server Message + return this._handleQemuAudioEvent(); + default: this._fail("Unexpected server message (type " + msgType + ")"); Log.Debug("sock.rQpeekBytes(30): " + this._sock.rQpeekBytes(30)); @@ -2683,6 +2694,13 @@ export default class RFB extends EventTargetMixin { this._qemuExtKeyEventSupported = true; return true; + case encodings.pseudoEncodingQEMUAudioEvent: + if (!this._qemuAudioSupported) { + RFB.messages.enableQemuAudioUpdates(this._sock, this._audio_channels, this._audio_sample_rate); + this._qemuAudioSupported = true; + } + return true; + case encodings.pseudoEncodingDesktopName: return this._handleDesktopName(); @@ -2705,6 +2723,93 @@ export default class RFB extends EventTargetMixin { } } + _handleQemuAudioEvent() { + if (this._sock.rQwait("Qemu Audio Event", 3, 1)) { + return false; + } + + const submsg = this._sock.rQshift8(); + if (submsg !== 1) { + Log.Warn("The given qemu message type " + submsg + " is not supported."); + return false; + } + + const operation = this._sock.rQshift16(); + + switch (operation) { + case 0: { + this._audio_context = null; + this._audio_next_start = 0; + return true; + } + case 1: { + this._audio_context = new AudioContext({ + latencyHint: "interactive", + sampleRate: this._audio_sample_rate, + }); + this._audio_next_start = 0; + return true; + } + case 2: break; + default: { + Log.Warn("The given qemu audio opertaion " + opertaion + " is not supported."); + return false; + } + } + + if (this._sock.rQwait("Qemu Audio payload length", 4, 4)) { + return false; + } + + const length = this._sock.rQshift32(); + + if (this._sock.rQwait("audio payload", length, 8)) { + return false; + } + + if (length !== 0) { + let payload = this._sock.rQshiftBytes(length, false); + + if (this._audio_context === null) { + return false; + } + + let sample_bytes = 2*this._audio_channels; + let buffer = this._audio_context.createBuffer(this._audio_channels, length/sample_bytes, this._audio_sample_rate); + + for (let ch = 0; ch < this._audio_channels; ch++) { + const channel = buffer.getChannelData(ch); + let channel_offset = ch*2; + for (let i = 0; i < buffer.length; i++) { + let p = i*sample_bytes + channel_offset; + let value = payload[p] + payload[p+1]*256; + channel[i] = (value / 32768.0) - 1.0; + } + } + + if (this._page_had_user_interaction) { + let ctime = this._audio_context.currentTime; + if (ctime > this._audio_next_start) { + this._audio_next_start = ctime; + } + let start_time = this._audio_next_start; + + this._audio_next_start += buffer.duration; + + let source = this._audio_context.createBufferSource(); + source.buffer = buffer; + source.connect(this._audio_context.destination); + source.start(start_time); + } + } + + return true; + } + + allow_audio() { + this._page_had_user_interaction = true; + } + _handleVMwareCursor() { const hotx = this._FBU.x; // hotspot-x const hoty = this._FBU.y; // hotspot-y @@ -3314,6 +3419,22 @@ RFB.messages = { sock.flush(); }, + enableQemuAudioUpdates(sock, channels, sample_rate) { + + sock.sQpush8(255); // msg-type + sock.sQpush8(1); // submessage-type + sock.sQpush16(2); // set sample format + sock.sQpush8(2); // format U16 + sock.sQpush8(channels); + sock.sQpush32(sample_rate); // audio frequency + + sock.sQpush8(255); // msg-type + sock.sQpush8(1); // submessage-type + sock.sQpush16(0); // enable audio + + sock.flush(); + }, + pixelFormat(sock, depth, trueColor) { let bpp; From 6f8f4bff62bb693f86a270d6e30a060987807557 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 3 Apr 2025 10:48:55 +0200 Subject: [PATCH 2/4] app: add enable audio setting Signed-off-by: Dietmar Maurer --- app/ui.js | 11 +++++++++++ core/rfb.js | 24 +++++++++++++++++++++++- vnc.html | 7 +++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/app/ui.js b/app/ui.js index f45a1635..3ecaf7a1 100644 --- a/app/ui.js +++ b/app/ui.js @@ -189,6 +189,7 @@ const UI = { UI.initSetting('repeaterID', ''); UI.initSetting('reconnect', false); UI.initSetting('reconnect_delay', 5000); + UI.initSetting('enable_audio', true); }, // Adds a link to the label elements on the corresponding input elements setupSettingLabels() { @@ -385,6 +386,8 @@ const UI = { UI.addSettingChangeHandler('logging', UI.updateLogging); UI.addSettingChangeHandler('reconnect'); UI.addSettingChangeHandler('reconnect_delay'); + UI.addSettingChangeHandler('enable_audio'); + UI.addSettingChangeHandler('enable_audio', UI.updateEnableAudio); }, addFullscreenHandlers() { @@ -898,6 +901,7 @@ const UI = { UI.updateSetting('logging'); UI.updateSetting('reconnect'); UI.updateSetting('reconnect_delay'); + UI.updateSetting('enable_audio'); document.getElementById('noVNC_settings') .classList.add("noVNC_open"); @@ -1109,6 +1113,8 @@ const UI = { UI.rfb.showDotCursor = UI.getSetting('show_dot'); UI.updateViewOnly(); // requires UI.rfb + UI.updateEnableAudio(); // requires UI.rfb + }, disconnect() { @@ -1801,6 +1807,11 @@ const UI = { selectbox.options.add(optn); }, + updateEnableAudio() { + if (!UI.rfb) return; + UI.rfb.enable_audio(UI.getSetting('enable_audio')); + }, + /* ------^------- * /MISC * ============== diff --git a/core/rfb.js b/core/rfb.js index 561bc4a7..4f5a51d2 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -156,6 +156,7 @@ export default class RFB extends EventTargetMixin { this._qemuAudioSupported = false; this._page_had_user_interaction = false; + this._audio_enable = false; this._audio_next_start = 0; this._audio_sample_rate = 44100; this._audio_channels = 2; @@ -2787,7 +2788,7 @@ export default class RFB extends EventTargetMixin { } } - if (this._page_had_user_interaction) { + if (this._page_had_user_interaction && this._audio_enable) { let ctime = this._audio_context.currentTime; if (ctime > this._audio_next_start) { this._audio_next_start = ctime; @@ -2806,6 +2807,19 @@ export default class RFB extends EventTargetMixin { return true; } + enable_audio(value) { + if (this._audio_enable !== value) { + this._audio_enable = value; + if (this._qemuAudioSupported) { + if (this._audio_enable) { + RFB.messages.enableQemuAudioUpdates(this._sock, this._audio_channels, this._audio_sample_rate); + } else { + RFB.messages.disableQemuAudioUpdates(this._sock); + } + } + } + } + allow_audio() { this._page_had_user_interaction = true; } @@ -3419,6 +3433,14 @@ RFB.messages = { sock.flush(); }, + disableQemuAudioUpdates(sock, channels, sample_rate) { + sock.sQpush8(255); // msg-type + sock.sQpush8(1); // submessage-type + sock.sQpush16(1); // disable audio + + sock.flush(); + }, + enableQemuAudioUpdates(sock, channels, sample_rate) { sock.sQpush8(255); // msg-type diff --git a/vnc.html b/vnc.html index 82cacd58..f6cab3f9 100644 --- a/vnc.html +++ b/vnc.html @@ -219,6 +219,13 @@ View only +
  • + +