[HTML5] Run Audio process in thread when available

This should fix some of the audio stuttering issues when the HTML5
export is compiled with threads support.
The API should be ported to AudioWorklet to (hopefully) be perfect.
That though, cannot be backported to 3.2 due to extra restriction of
AudioWorklet (which only runs in SecureContext, and needs a polyfill for
Safari).

platform/javascript/SCsub

@@ -18,6 +18,7 @@ build = env.add_program(build_targets, javascript_files)
 
 js_libraries = [
     "native/http_request.js",
+    "native/library_godot_audio.js",
 ]
 for lib in js_libraries:
     env.Append(LINKFLAGS=["--js-library", env.File(lib).path])

platform/javascript/audio_driver_javascript.cpp

@@ -34,31 +34,55 @@
 
 #include <emscripten.h>
 
+#include "godot_audio.h"
+
 AudioDriverJavaScript *AudioDriverJavaScript::singleton = NULL;
 
 bool AudioDriverJavaScript::is_available() {
-	return EM_ASM_INT({
-		if (!(window.AudioContext || window.webkitAudioContext)) {
-			return 0;
-		}
-		return 1;
-	}) != 0;
+	return godot_audio_is_available() != 0;
 }
 
 const char *AudioDriverJavaScript::get_name() const {
 	return "JavaScript";
 }
 
-extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() {
-	AudioDriverJavaScript::singleton->mix_to_js();
+#ifndef NO_THREADS
+void AudioDriverJavaScript::_audio_thread_func(void *p_data) {
+	AudioDriverJavaScript *obj = static_cast<AudioDriverJavaScript *>(p_data);
+	while (!obj->quit) {
+		obj->lock();
+		if (!obj->needs_process) {
+			obj->unlock();
+			OS::get_singleton()->delay_usec(1000); // Give the browser some slack.
+			continue;
+		}
+		obj->_js_driver_process();
+		obj->needs_process = false;
+		obj->unlock();
+	}
+}
+#endif
+
+extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_start() {
+#ifndef NO_THREADS
+	AudioDriverJavaScript::singleton->lock();
+#else
+	AudioDriverJavaScript::singleton->_js_driver_process();
+#endif
+}
+
+extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_end() {
+#ifndef NO_THREADS
+	AudioDriverJavaScript::singleton->needs_process = true;
+	AudioDriverJavaScript::singleton->unlock();
+#endif
 }
 
 extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) {
 	AudioDriverJavaScript::singleton->process_capture(sample);
 }
 
-void AudioDriverJavaScript::mix_to_js() {
-	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
+void AudioDriverJavaScript::_js_driver_process() {
 	int sample_count = memarr_len(internal_buffer) / channel_count;
 	int32_t *stream_buffer = reinterpret_cast<int32_t *>(internal_buffer);
 	audio_server_process(sample_count, stream_buffer);
@@ -73,37 +97,12 @@ void AudioDriverJavaScript::process_capture(float sample) {
 }
 
 Error AudioDriverJavaScript::init() {
-	int mix_rate = GLOBAL_GET("audio/mix_rate");
+	mix_rate = GLOBAL_GET("audio/mix_rate");
 	int latency = GLOBAL_GET("audio/output_latency");
 
-	/* clang-format off */
-	_driver_id = EM_ASM_INT({
-		const MIX_RATE = $0;
-		const LATENCY = $1 / 1000;
-		return Module.IDHandler.add({
-			'context': new (window.AudioContext || window.webkitAudioContext)({ sampleRate: MIX_RATE, latencyHint: LATENCY}),
-			'input': null,
-			'stream': null,
-			'script': null
-		});
-	}, mix_rate, latency);
-	/* clang-format on */
-
-	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
+	channel_count = godot_audio_init(mix_rate, latency);
 	buffer_length = closest_power_of_2((latency * mix_rate / 1000) * channel_count);
-	/* clang-format off */
-	buffer_length = EM_ASM_INT({
-		var ref = Module.IDHandler.get($0);
-		const ctx = ref['context'];
-		const BUFFER_LENGTH = $1;
-		const CHANNEL_COUNT = $2;
-
-		var script = ctx.createScriptProcessor(BUFFER_LENGTH, 2, CHANNEL_COUNT);
-		script.connect(ctx.destination);
-		ref['script'] = script;
-		return script.bufferSize;
-	}, _driver_id, buffer_length, channel_count);
-	/* clang-format on */
+	buffer_length = godot_audio_create_processor(buffer_length, channel_count);
 	if (!buffer_length) {
 		return FAILED;
 	}
@@ -114,134 +113,67 @@ Error AudioDriverJavaScript::init() {
 		internal_buffer = memnew_arr(float, buffer_length *channel_count);
 	}
 
-	return internal_buffer ? OK : ERR_OUT_OF_MEMORY;
+	if (!internal_buffer) {
+		return ERR_OUT_OF_MEMORY;
+	}
+	return OK;
 }
 
 void AudioDriverJavaScript::start() {
-	/* clang-format off */
-	EM_ASM({
-		const ref = Module.IDHandler.get($0);
-		var INTERNAL_BUFFER_PTR = $1;
-
-		var audioDriverMixFunction = cwrap('audio_driver_js_mix');
-		var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
-		ref['script'].onaudioprocess = function(audioProcessingEvent) {
-			audioDriverMixFunction();
-
-			var input = audioProcessingEvent.inputBuffer;
-			var output = audioProcessingEvent.outputBuffer;
-			var internalBuffer = HEAPF32.subarray(
-					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT,
-					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
-
-			for (var channel = 0; channel < output.numberOfChannels; channel++) {
-				var outputData = output.getChannelData(channel);
-				// Loop through samples.
-				for (var sample = 0; sample < outputData.length; sample++) {
-					outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
-				}
-			}
-
-			if (ref['input']) {
-				var inputDataL = input.getChannelData(0);
-				var inputDataR = input.getChannelData(1);
-				for (var i = 0; i < inputDataL.length; i++) {
-					audioDriverProcessCapture(inputDataL[i]);
-					audioDriverProcessCapture(inputDataR[i]);
-				}
-			}
-		};
-	}, _driver_id, internal_buffer);
-	/* clang-format on */
+#ifndef NO_THREADS
+	mutex = Mutex::create();
+	thread = Thread::create(_audio_thread_func, this);
+#endif
+	godot_audio_start(internal_buffer);
 }
 
 void AudioDriverJavaScript::resume() {
-	/* clang-format off */
-	EM_ASM({
-		const ref = Module.IDHandler.get($0);
-		if (ref && ref['context'] && ref['context'].resume)
-			ref['context'].resume();
-	}, _driver_id);
-	/* clang-format on */
+	godot_audio_resume();
 }
 
 float AudioDriverJavaScript::get_latency() {
-	/* clang-format off */
-	return EM_ASM_DOUBLE({
-		const ref = Module.IDHandler.get($0);
-		var latency = 0;
-		if (ref && ref['context']) {
-			const ctx = ref['context'];
-			if (ctx.baseLatency) {
-				latency += ctx.baseLatency;
-			}
-			if (ctx.outputLatency) {
-				latency += ctx.outputLatency;
-			}
-		}
-		return latency;
-	}, _driver_id);
-	/* clang-format on */
+	return godot_audio_get_latency();
 }
 
 int AudioDriverJavaScript::get_mix_rate() const {
-	/* clang-format off */
-	return EM_ASM_INT({
-		const ref = Module.IDHandler.get($0);
-		return ref && ref['context'] ? ref['context'].sampleRate : 0;
-	}, _driver_id);
-	/* clang-format on */
+	return mix_rate;
 }
 
 AudioDriver::SpeakerMode AudioDriverJavaScript::get_speaker_mode() const {
-	/* clang-format off */
-	return get_speaker_mode_by_total_channels(EM_ASM_INT({
-		const ref = Module.IDHandler.get($0);
-		return ref && ref['context'] ? ref['context'].destination.channelCount : 0;
-	}, _driver_id));
-	/* clang-format on */
+	return get_speaker_mode_by_total_channels(channel_count);
 }
 
-// No locking, as threads are not supported.
 void AudioDriverJavaScript::lock() {
+#ifndef NO_THREADS
+	if (mutex) {
+		mutex->lock();
+	}
+#endif
 }
 
 void AudioDriverJavaScript::unlock() {
+#ifndef NO_THREADS
+	if (mutex) {
+		mutex->unlock();
+	}
+#endif
 }
 
 void AudioDriverJavaScript::finish_async() {
-	// Close the context, add the operation to the async_finish list in module.
-	int id = _driver_id;
-	_driver_id = 0;
-
-	/* clang-format off */
-	EM_ASM({
-		const id = $0;
-		var ref = Module.IDHandler.get(id);
-		Module.async_finish.push(new Promise(function(accept, reject) {
-			if (!ref) {
-				console.log("Ref not found!", id, Module.IDHandler);
-				setTimeout(accept, 0);
-			} else {
-				Module.IDHandler.remove(id);
-				const context = ref['context'];
-				// Disconnect script and input.
-				ref['script'].disconnect();
-				if (ref['input'])
-					ref['input'].disconnect();
-				ref = null;
-				context.close().then(function() {
-					accept();
-				}).catch(function(e) {
-					accept();
-				});
-			}
-		}));
-	}, id);
-	/* clang-format on */
+#ifndef NO_THREADS
+	quit = true; // Ask thread to quit.
+#endif
+	godot_audio_finish_async();
 }
 
 void AudioDriverJavaScript::finish() {
+#ifndef NO_THREADS
+	Thread::wait_to_finish(thread);
+	memdelete(thread);
+	thread = NULL;
+	memdelete(mutex);
+	mutex = NULL;
+#endif
 	if (internal_buffer) {
 		memdelete_arr(internal_buffer);
 		internal_buffer = NULL;
@@ -250,62 +182,28 @@ void AudioDriverJavaScript::finish() {
 
 Error AudioDriverJavaScript::capture_start() {
 	input_buffer_init(buffer_length);
-
-	/* clang-format off */
-	EM_ASM({
-		function gotMediaInput(stream) {
-			var ref = Module.IDHandler.get($0);
-			ref['stream'] = stream;
-			ref['input'] = ref['context'].createMediaStreamSource(stream);
-			ref['input'].connect(ref['script']);
-		}
-
-		function gotMediaInputError(e) {
-			out(e);
-		}
-
-		if (navigator.mediaDevices.getUserMedia) {
-			navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
-		} else {
-			if (!navigator.getUserMedia)
-				navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
-			navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
-		}
-	}, _driver_id);
-	/* clang-format on */
-
+	godot_audio_capture_start();
 	return OK;
 }
 
 Error AudioDriverJavaScript::capture_stop() {
-	/* clang-format off */
-	EM_ASM({
-		var ref = Module.IDHandler.get($0);
-		if (ref['stream']) {
-			const tracks = ref['stream'].getTracks();
-			for (var i = 0; i < tracks.length; i++) {
-				tracks[i].stop();
-			}
-			ref['stream'] = null;
-		}
-
-		if (ref['input']) {
-			ref['input'].disconnect();
-			ref['input'] = null;
-		}
-
-	}, _driver_id);
-	/* clang-format on */
-
+	godot_audio_capture_stop();
 	input_buffer.clear();
-
 	return OK;
 }
 
 AudioDriverJavaScript::AudioDriverJavaScript() {
-	_driver_id = 0;
 	internal_buffer = NULL;
 	buffer_length = 0;
+	mix_rate = 0;
+	channel_count = 0;
+
+#ifndef NO_THREADS
+	mutex = NULL;
+	thread = NULL;
+	quit = false;
+	needs_process = true;
+#endif
 
 	singleton = this;
 }

platform/javascript/audio_driver_javascript.h

@@ -31,18 +31,33 @@
 #ifndef AUDIO_DRIVER_JAVASCRIPT_H
 #define AUDIO_DRIVER_JAVASCRIPT_H
 
+#include "core/os/mutex.h"
+#include "core/os/thread.h"
 #include "servers/audio_server.h"
 
 class AudioDriverJavaScript : public AudioDriver {
 
+private:
 	float *internal_buffer;
 
-	int _driver_id;
 	int buffer_length;
 
+	int mix_rate;
+	int channel_count;
+
 public:
+#ifndef NO_THREADS
+	Mutex *mutex;
+	Thread *thread;
+	bool quit;
+	bool needs_process;
+
+	static void _audio_thread_func(void *p_data);
+#endif
+
+	void _js_driver_process();
+
 	static bool is_available();
-	void mix_to_js();
 	void process_capture(float sample);
 
 	static AudioDriverJavaScript *singleton;

platform/javascript/godot_audio.h

@@ -0,0 +1,58 @@
+/*************************************************************************/
+/*  godot_audio.h                                                        */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef GODOT_AUDIO_H
+#define GODOT_AUDIO_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "stddef.h"
+
+extern int godot_audio_is_available();
+
+extern int godot_audio_init(int p_mix_rate, int p_latency);
+extern int godot_audio_create_processor(int p_buffer_length, int p_channel_count);
+
+extern void godot_audio_start(float *r_buffer_ptr);
+extern void godot_audio_resume();
+extern void godot_audio_finish_async();
+
+extern float godot_audio_get_latency();
+
+extern void godot_audio_capture_start();
+extern void godot_audio_capture_stop();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GODOT_AUDIO_H */

platform/javascript/native/library_godot_audio.js

@@ -0,0 +1,173 @@
+/*************************************************************************/
+/*  library_godot_audio.js                                               */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+var GodotAudio = {
+
+	$GodotAudio: {
+
+		ctx: null,
+		input: null,
+		script: null,
+	},
+
+	godot_audio_is_available__proxy: 'sync',
+	godot_audio_is_available: function () {
+		if (!(window.AudioContext || window.webkitAudioContext)) {
+			return 0;
+		}
+		return 1;
+	},
+
+	godot_audio_init: function(mix_rate, latency) {
+		GodotAudio.ctx = new (window.AudioContext || window.webkitAudioContext)({
+			sampleRate: mix_rate,
+			latencyHint: latency
+		});
+		return GodotAudio.ctx.destination.channelCount;
+	},
+
+	godot_audio_create_processor: function(buffer_length, channel_count) {
+		GodotAudio.script = GodotAudio.ctx.createScriptProcessor(buffer_length, 2, channel_count);
+		GodotAudio.script.connect(GodotAudio.ctx.destination);
+		return GodotAudio.script.bufferSize;
+	},
+
+	godot_audio_start: function(buffer_ptr) {
+		var audioDriverProcessStart = cwrap('audio_driver_process_start');
+		var audioDriverProcessEnd = cwrap('audio_driver_process_end');
+		var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
+		GodotAudio.script.onaudioprocess = function(audioProcessingEvent) {
+			audioDriverProcessStart();
+
+			var input = audioProcessingEvent.inputBuffer;
+			var output = audioProcessingEvent.outputBuffer;
+			var internalBuffer = HEAPF32.subarray(
+					buffer_ptr / HEAPF32.BYTES_PER_ELEMENT,
+					buffer_ptr / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
+			for (var channel = 0; channel < output.numberOfChannels; channel++) {
+				var outputData = output.getChannelData(channel);
+				// Loop through samples.
+				for (var sample = 0; sample < outputData.length; sample++) {
+					outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
+				}
+			}
+
+			if (GodotAudio.input) {
+				var inputDataL = input.getChannelData(0);
+				var inputDataR = input.getChannelData(1);
+				for (var i = 0; i < inputDataL.length; i++) {
+					audioDriverProcessCapture(inputDataL[i]);
+					audioDriverProcessCapture(inputDataR[i]);
+				}
+			}
+			audioDriverProcessEnd();
+		};
+	},
+
+	godot_audio_resume: function() {
+		if (GodotAudio.ctx && GodotAudio.ctx.state != 'running') {
+			GodotAudio.ctx.resume();
+		}
+	},
+
+	godot_audio_finish_async: function() {
+		Module.async_finish.push(new Promise(function(accept, reject) {
+			if (!GodotAudio.ctx) {
+				setTimeout(accept, 0);
+			} else {
+				if (GodotAudio.script) {
+					GodotAudio.script.disconnect();
+					GodotAudio.script = null;
+				}
+				if (GodotAudio.input) {
+					GodotAudio.input.disconnect();
+					GodotAudio.input = null;
+				}
+				GodotAudio.ctx.close().then(function() {
+					accept();
+				}).catch(function(e) {
+					accept();
+				});
+				GodotAudio.ctx = null;
+			}
+		}));
+	},
+
+	godot_audio_get_latency__proxy: 'sync',
+	godot_audio_get_latency: function() {
+		var latency = 0;
+		if (GodotAudio.ctx) {
+			if (GodotAudio.ctx.baseLatency) {
+				latency += GodotAudio.ctx.baseLatency;
+			}
+			if (GodotAudio.ctx.outputLatency) {
+				latency += GodotAudio.ctx.outputLatency;
+			}
+		}
+		return latency;
+	},
+
+	godot_audio_capture_start__proxy: 'sync',
+	godot_audio_capture_start: function() {
+		if (GodotAudio.input) {
+			return; // Already started.
+		}
+		function gotMediaInput(stream) {
+			GodotAudio.input = GodotAudio.ctx.createMediaStreamSource(stream);
+			GodotAudio.input.connect(GodotAudio.script);
+		}
+
+		function gotMediaInputError(e) {
+			out(e);
+		}
+
+		if (navigator.mediaDevices.getUserMedia) {
+			navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
+		} else {
+			if (!navigator.getUserMedia)
+				navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
+			navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
+		}
+	},
+
+	godot_audio_capture_stop__proxy: 'sync',
+	godot_audio_capture_stop: function() {
+		if (GodotAudio.input) {
+			const tracks = GodotAudio.input.mediaStream.getTracks();
+			for (var i = 0; i < tracks.length; i++) {
+				tracks[i].stop();
+			}
+			GodotAudio.input.disconnect();
+			GodotAudio.input = null;
+		}
+	},
+};
+
+autoAddDeps(GodotAudio, "$GodotAudio");
+mergeInto(LibraryManager.library, GodotAudio);