Merge pull request #42505 from Faless/js/4.0_audio_threads

[HTML5] Move audio processing to thread when threads are enabled.

platform/javascript/SCsub

@@ -19,6 +19,7 @@ build = env.add_program(build_targets, javascript_files)
 
 js_libraries = [
     "native/http_request.js",
+    "native/library_godot_audio.js",
 ]
 for lib in js_libraries:
     env.Append(LINKFLAGS=["--js-library", env.File(lib).path])

platform/javascript/audio_driver_javascript.cpp

@@ -31,34 +31,57 @@
 #include "audio_driver_javascript.h"
 
 #include "core/project_settings.h"
+#include "godot_audio.h"
 
 #include <emscripten.h>
 
 AudioDriverJavaScript *AudioDriverJavaScript::singleton = nullptr;
 
 bool AudioDriverJavaScript::is_available() {
-	return EM_ASM_INT({
-		if (!(window.AudioContext || window.webkitAudioContext)) {
-			return 0;
-		}
-		return 1;
-	}) != 0;
+	return godot_audio_is_available() != 0;
 }
 
 const char *AudioDriverJavaScript::get_name() const {
 	return "JavaScript";
 }
 
-extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() {
-	AudioDriverJavaScript::singleton->mix_to_js();
+#ifndef NO_THREADS
+void AudioDriverJavaScript::_audio_thread_func(void *p_data) {
+	AudioDriverJavaScript *obj = static_cast<AudioDriverJavaScript *>(p_data);
+	while (!obj->quit) {
+		obj->lock();
+		if (!obj->needs_process) {
+			obj->unlock();
+			OS::get_singleton()->delay_usec(1000); // Give the browser some slack.
+			continue;
+		}
+		obj->_js_driver_process();
+		obj->needs_process = false;
+		obj->unlock();
+	}
+}
+#endif
+
+extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_start() {
+#ifndef NO_THREADS
+	AudioDriverJavaScript::singleton->lock();
+#else
+	AudioDriverJavaScript::singleton->_js_driver_process();
+#endif
+}
+
+extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_end() {
+#ifndef NO_THREADS
+	AudioDriverJavaScript::singleton->needs_process = true;
+	AudioDriverJavaScript::singleton->unlock();
+#endif
 }
 
 extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) {
 	AudioDriverJavaScript::singleton->process_capture(sample);
 }
 
-void AudioDriverJavaScript::mix_to_js() {
-	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
+void AudioDriverJavaScript::_js_driver_process() {
 	int sample_count = memarr_len(internal_buffer) / channel_count;
 	int32_t *stream_buffer = reinterpret_cast<int32_t *>(internal_buffer);
 	audio_server_process(sample_count, stream_buffer);
@@ -73,37 +96,12 @@ void AudioDriverJavaScript::process_capture(float sample) {
 }
 
 Error AudioDriverJavaScript::init() {
-	int mix_rate = GLOBAL_GET("audio/mix_rate");
+	mix_rate = GLOBAL_GET("audio/mix_rate");
 	int latency = GLOBAL_GET("audio/output_latency");
 
-	/* clang-format off */
-	_driver_id = EM_ASM_INT({
-		const MIX_RATE = $0;
-		const LATENCY = $1 / 1000;
-		return Module.IDHandler.add({
-			'context': new (window.AudioContext || window.webkitAudioContext)({ sampleRate: MIX_RATE, latencyHint: LATENCY}),
-			'input': null,
-			'stream': null,
-			'script': null
-		});
-	}, mix_rate, latency);
-	/* clang-format on */
-
-	int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
+	channel_count = godot_audio_init(mix_rate, latency);
 	buffer_length = closest_power_of_2((latency * mix_rate / 1000) * channel_count);
-	/* clang-format off */
-	buffer_length = EM_ASM_INT({
-		var ref = Module.IDHandler.get($0);
-		const ctx = ref['context'];
-		const BUFFER_LENGTH = $1;
-		const CHANNEL_COUNT = $2;
-
-		var script = ctx.createScriptProcessor(BUFFER_LENGTH, 2, CHANNEL_COUNT);
-		script.connect(ctx.destination);
-		ref['script'] = script;
-		return script.bufferSize;
-	}, _driver_id, buffer_length, channel_count);
-	/* clang-format on */
+	buffer_length = godot_audio_create_processor(buffer_length, channel_count);
 	if (!buffer_length) {
 		return FAILED;
 	}
@@ -114,134 +112,60 @@ Error AudioDriverJavaScript::init() {
 		internal_buffer = memnew_arr(float, buffer_length *channel_count);
 	}
 
-	return internal_buffer ? OK : ERR_OUT_OF_MEMORY;
+	if (!internal_buffer) {
+		return ERR_OUT_OF_MEMORY;
+	}
+	return OK;
 }
 
 void AudioDriverJavaScript::start() {
-	/* clang-format off */
-	EM_ASM({
-		const ref = Module.IDHandler.get($0);
-		var INTERNAL_BUFFER_PTR = $1;
-
-		var audioDriverMixFunction = cwrap('audio_driver_js_mix');
-		var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
-		ref['script'].onaudioprocess = function(audioProcessingEvent) {
-			audioDriverMixFunction();
-
-			var input = audioProcessingEvent.inputBuffer;
-			var output = audioProcessingEvent.outputBuffer;
-			var internalBuffer = HEAPF32.subarray(
-					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT,
-					INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
-
-			for (var channel = 0; channel < output.numberOfChannels; channel++) {
-				var outputData = output.getChannelData(channel);
-				// Loop through samples.
-				for (var sample = 0; sample < outputData.length; sample++) {
-					outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
-				}
-			}
-
-			if (ref['input']) {
-				var inputDataL = input.getChannelData(0);
-				var inputDataR = input.getChannelData(1);
-				for (var i = 0; i < inputDataL.length; i++) {
-					audioDriverProcessCapture(inputDataL[i]);
-					audioDriverProcessCapture(inputDataR[i]);
-				}
-			}
-		};
-	}, _driver_id, internal_buffer);
-	/* clang-format on */
+#ifndef NO_THREADS
+	thread = Thread::create(_audio_thread_func, this);
+#endif
+	godot_audio_start(internal_buffer);
 }
 
 void AudioDriverJavaScript::resume() {
-	/* clang-format off */
-	EM_ASM({
-		const ref = Module.IDHandler.get($0);
-		if (ref && ref['context'] && ref['context'].resume)
-			ref['context'].resume();
-	}, _driver_id);
-	/* clang-format on */
+	godot_audio_resume();
 }
 
 float AudioDriverJavaScript::get_latency() {
-	/* clang-format off */
-	return EM_ASM_DOUBLE({
-		const ref = Module.IDHandler.get($0);
-		var latency = 0;
-		if (ref && ref['context']) {
-			const ctx = ref['context'];
-			if (ctx.baseLatency) {
-				latency += ctx.baseLatency;
-			}
-			if (ctx.outputLatency) {
-				latency += ctx.outputLatency;
-			}
-		}
-		return latency;
-	}, _driver_id);
-	/* clang-format on */
+	return godot_audio_get_latency();
 }
 
 int AudioDriverJavaScript::get_mix_rate() const {
-	/* clang-format off */
-	return EM_ASM_INT({
-		const ref = Module.IDHandler.get($0);
-		return ref && ref['context'] ? ref['context'].sampleRate : 0;
-	}, _driver_id);
-	/* clang-format on */
+	return mix_rate;
 }
 
 AudioDriver::SpeakerMode AudioDriverJavaScript::get_speaker_mode() const {
-	/* clang-format off */
-	return get_speaker_mode_by_total_channels(EM_ASM_INT({
-		const ref = Module.IDHandler.get($0);
-		return ref && ref['context'] ? ref['context'].destination.channelCount : 0;
-	}, _driver_id));
-	/* clang-format on */
+	return get_speaker_mode_by_total_channels(channel_count);
 }
 
-// No locking, as threads are not supported.
 void AudioDriverJavaScript::lock() {
+#ifndef NO_THREADS
+	mutex.lock();
+#endif
 }
 
 void AudioDriverJavaScript::unlock() {
+#ifndef NO_THREADS
+	mutex.unlock();
+#endif
 }
 
 void AudioDriverJavaScript::finish_async() {
-	// Close the context, add the operation to the async_finish list in module.
-	int id = _driver_id;
-	_driver_id = 0;
-
-	/* clang-format off */
-	EM_ASM({
-		const id = $0;
-		var ref = Module.IDHandler.get(id);
-		Module.async_finish.push(new Promise(function(accept, reject) {
-			if (!ref) {
-				console.log("Ref not found!", id, Module.IDHandler);
-				setTimeout(accept, 0);
-			} else {
-				Module.IDHandler.remove(id);
-				const context = ref['context'];
-				// Disconnect script and input.
-				ref['script'].disconnect();
-				if (ref['input'])
-					ref['input'].disconnect();
-				ref = null;
-				context.close().then(function() {
-					accept();
-				}).catch(function(e) {
-					accept();
-				});
-			}
-		}));
-	}, id);
-	/* clang-format on */
+#ifndef NO_THREADS
+	quit = true; // Ask thread to quit.
+#endif
+	godot_audio_finish_async();
 }
 
 void AudioDriverJavaScript::finish() {
+#ifndef NO_THREADS
+	Thread::wait_to_finish(thread);
+	memdelete(thread);
+	thread = NULL;
+#endif
 	if (internal_buffer) {
 		memdelete_arr(internal_buffer);
 		internal_buffer = nullptr;
@@ -249,56 +173,14 @@ void AudioDriverJavaScript::finish() {
 }
 
 Error AudioDriverJavaScript::capture_start() {
+	godot_audio_capture_stop();
 	input_buffer_init(buffer_length);
-
-	/* clang-format off */
-	EM_ASM({
-		function gotMediaInput(stream) {
-			var ref = Module.IDHandler.get($0);
-			ref['stream'] = stream;
-			ref['input'] = ref['context'].createMediaStreamSource(stream);
-			ref['input'].connect(ref['script']);
-		}
-
-		function gotMediaInputError(e) {
-			out(e);
-		}
-
-		if (navigator.mediaDevices.getUserMedia) {
-			navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
-		} else {
-			if (!navigator.getUserMedia)
-				navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
-			navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
-		}
-	}, _driver_id);
-	/* clang-format on */
-
+	godot_audio_capture_start();
 	return OK;
 }
 
 Error AudioDriverJavaScript::capture_stop() {
-	/* clang-format off */
-	EM_ASM({
-		var ref = Module.IDHandler.get($0);
-		if (ref['stream']) {
-			const tracks = ref['stream'].getTracks();
-			for (var i = 0; i < tracks.length; i++) {
-				tracks[i].stop();
-			}
-			ref['stream'] = null;
-		}
-
-		if (ref['input']) {
-			ref['input'].disconnect();
-			ref['input'] = null;
-		}
-
-	}, _driver_id);
-	/* clang-format on */
-
 	input_buffer.clear();
-
 	return OK;
 }
 

platform/javascript/audio_driver_javascript.h

@@ -33,15 +33,30 @@
 
 #include "servers/audio_server.h"
 
+#include "core/os/mutex.h"
+#include "core/os/thread.h"
+
 class AudioDriverJavaScript : public AudioDriver {
+private:
 	float *internal_buffer = nullptr;
 
-	int _driver_id = 0;
 	int buffer_length = 0;
+	int mix_rate = 0;
+	int channel_count = 0;
 
 public:
+#ifndef NO_THREADS
+	Mutex mutex;
+	Thread *thread = nullptr;
+	bool quit = false;
+	bool needs_process = true;
+
+	static void _audio_thread_func(void *p_data);
+#endif
+
+	void _js_driver_process();
+
 	static bool is_available();
-	void mix_to_js();
 	void process_capture(float sample);
 
 	static AudioDriverJavaScript *singleton;

platform/javascript/detect.py

@@ -139,6 +139,7 @@ def configure(env):
         env.Append(LINKFLAGS=["-s", "USE_PTHREADS=1"])
         env.Append(LINKFLAGS=["-s", "PTHREAD_POOL_SIZE=4"])
         env.Append(LINKFLAGS=["-s", "WASM_MEM_MAX=2048MB"])
+        env.extra_suffix = ".threads" + env.extra_suffix
     else:
         env.Append(CPPDEFINES=["NO_THREADS"])
 

platform/javascript/export/export.cpp

@@ -124,6 +124,9 @@ public:
 		String s = "HTTP/1.1 200 OK\r\n";
 		s += "Connection: Close\r\n";
 		s += "Content-Type: " + ctype + "\r\n";
+		s += "Access-Control-Allow-Origin: *\r\n";
+		s += "Cross-Origin-Opener-Policy: same-origin\r\n";
+		s += "Cross-Origin-Embedder-Policy: require-corp\r\n";
 		s += "\r\n";
 		CharString cs = s.utf8();
 		Error err = connection->put_data((const uint8_t *)cs.get_data(), cs.size() - 1);

platform/javascript/godot_audio.h

@@ -0,0 +1,58 @@
+/*************************************************************************/
+/*  godot_audio.h                                                        */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef GODOT_AUDIO_H
+#define GODOT_AUDIO_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "stddef.h"
+
+extern int godot_audio_is_available();
+
+extern int godot_audio_init(int p_mix_rate, int p_latency);
+extern int godot_audio_create_processor(int p_buffer_length, int p_channel_count);
+
+extern void godot_audio_start(float *r_buffer_ptr);
+extern void godot_audio_resume();
+extern void godot_audio_finish_async();
+
+extern float godot_audio_get_latency();
+
+extern void godot_audio_capture_start();
+extern void godot_audio_capture_stop();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GODOT_AUDIO_H */

platform/javascript/native/library_godot_audio.js

@@ -0,0 +1,173 @@
+/*************************************************************************/
+/*  library_godot_audio.js                                               */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+var GodotAudio = {
+
+	$GodotAudio: {
+
+		ctx: null,
+		input: null,
+		script: null,
+	},
+
+	godot_audio_is_available__proxy: 'sync',
+	godot_audio_is_available: function () {
+		if (!(window.AudioContext || window.webkitAudioContext)) {
+			return 0;
+		}
+		return 1;
+	},
+
+	godot_audio_init: function(mix_rate, latency) {
+		GodotAudio.ctx = new (window.AudioContext || window.webkitAudioContext)({
+			sampleRate: mix_rate,
+			latencyHint: latency
+		});
+		return GodotAudio.ctx.destination.channelCount;
+	},
+
+	godot_audio_create_processor: function(buffer_length, channel_count) {
+		GodotAudio.script = GodotAudio.ctx.createScriptProcessor(buffer_length, 2, channel_count);
+		GodotAudio.script.connect(GodotAudio.ctx.destination);
+		return GodotAudio.script.bufferSize;
+	},
+
+	godot_audio_start: function(buffer_ptr) {
+		var audioDriverProcessStart = cwrap('audio_driver_process_start');
+		var audioDriverProcessEnd = cwrap('audio_driver_process_end');
+		var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
+		GodotAudio.script.onaudioprocess = function(audioProcessingEvent) {
+			audioDriverProcessStart();
+
+			var input = audioProcessingEvent.inputBuffer;
+			var output = audioProcessingEvent.outputBuffer;
+			var internalBuffer = HEAPF32.subarray(
+					buffer_ptr / HEAPF32.BYTES_PER_ELEMENT,
+					buffer_ptr / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
+			for (var channel = 0; channel < output.numberOfChannels; channel++) {
+				var outputData = output.getChannelData(channel);
+				// Loop through samples.
+				for (var sample = 0; sample < outputData.length; sample++) {
+					outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
+				}
+			}
+
+			if (GodotAudio.input) {
+				var inputDataL = input.getChannelData(0);
+				var inputDataR = input.getChannelData(1);
+				for (var i = 0; i < inputDataL.length; i++) {
+					audioDriverProcessCapture(inputDataL[i]);
+					audioDriverProcessCapture(inputDataR[i]);
+				}
+			}
+			audioDriverProcessEnd();
+		};
+	},
+
+	godot_audio_resume: function() {
+		if (GodotAudio.ctx && GodotAudio.ctx.state != 'running') {
+			GodotAudio.ctx.resume();
+		}
+	},
+
+	godot_audio_finish_async: function() {
+		Module.async_finish.push(new Promise(function(accept, reject) {
+			if (!GodotAudio.ctx) {
+				setTimeout(accept, 0);
+			} else {
+				if (GodotAudio.script) {
+					GodotAudio.script.disconnect();
+					GodotAudio.script = null;
+				}
+				if (GodotAudio.input) {
+					GodotAudio.input.disconnect();
+					GodotAudio.input = null;
+				}
+				GodotAudio.ctx.close().then(function() {
+					accept();
+				}).catch(function(e) {
+					accept();
+				});
+				GodotAudio.ctx = null;
+			}
+		}));
+	},
+
+	godot_audio_get_latency__proxy: 'sync',
+	godot_audio_get_latency: function() {
+		var latency = 0;
+		if (GodotAudio.ctx) {
+			if (GodotAudio.ctx.baseLatency) {
+				latency += GodotAudio.ctx.baseLatency;
+			}
+			if (GodotAudio.ctx.outputLatency) {
+				latency += GodotAudio.ctx.outputLatency;
+			}
+		}
+		return latency;
+	},
+
+	godot_audio_capture_start__proxy: 'sync',
+	godot_audio_capture_start: function() {
+		if (GodotAudio.input) {
+			return; // Already started.
+		}
+		function gotMediaInput(stream) {
+			GodotAudio.input = GodotAudio.ctx.createMediaStreamSource(stream);
+			GodotAudio.input.connect(GodotAudio.script);
+		}
+
+		function gotMediaInputError(e) {
+			out(e);
+		}
+
+		if (navigator.mediaDevices.getUserMedia) {
+			navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
+		} else {
+			if (!navigator.getUserMedia)
+				navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
+			navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
+		}
+	},
+
+	godot_audio_capture_stop__proxy: 'sync',
+	godot_audio_capture_stop: function() {
+		if (GodotAudio.input) {
+			const tracks = GodotAudio.input.mediaStream.getTracks();
+			for (var i = 0; i < tracks.length; i++) {
+				tracks[i].stop();
+			}
+			GodotAudio.input.disconnect();
+			GodotAudio.input = null;
+		}
+	},
+};
+
+autoAddDeps(GodotAudio, "$GodotAudio");
+mergeInto(LibraryManager.library, GodotAudio);