JS synchronous start, better persistent FS sync.

The engine now expects to emscripten FS to be setup and sync-ed before
main is called. This is exposed via `Module["initFS"]` which also allows
to setup multiple persistence paths (internal use only for now).

Additionally, FS syncing is done **once** for every loop if at least one
file in a persistent path was open for writing and closed, and if the FS
is not syncing already.

This should potentially fix issues reported by users where "autosave"
would not work on the web (never calling `syncfs` because of too many
writes).

platform/javascript/engine/engine.js

@@ -33,6 +33,7 @@ Function('return this')()['Engine'] = (function() {
 		this.resizeCanvasOnStart = false;
 		this.onExecute = null;
 		this.onExit = null;
+		this.persistentPaths = [];
 	};
 
 	Engine.prototype.init = /** @param {string=} basePath */ function(basePath) {
@@ -56,12 +57,14 @@ Function('return this')()['Engine'] = (function() {
 			config['locateFile'] = Utils.createLocateRewrite(loadPath);
 			config['instantiateWasm'] = Utils.createInstantiatePromise(loadPromise);
 			Godot(config).then(function(module) {
-				me.rtenv = module;
-				if (unloadAfterInit) {
-					unload();
-				}
-				resolve();
-				config = null;
+				module['initFS'](me.persistentPaths).then(function(fs_err) {
+					me.rtenv = module;
+					if (unloadAfterInit) {
+						unload();
+					}
+					resolve();
+					config = null;
+				});
 			});
 		});
 		return initPromise;
@@ -220,6 +223,10 @@ Function('return this')()['Engine'] = (function() {
 		this.rtenv['copyToFS'](path, buffer);
 	};
 
+	Engine.prototype.setPersistentPaths = function(persistentPaths) {
+		this.persistentPaths = persistentPaths;
+	};
+
 	// Closure compiler exported engine methods.
 	/** @export */
 	Engine['isWebGLAvailable'] = Utils.isWebGLAvailable;
@@ -241,5 +248,6 @@ Function('return this')()['Engine'] = (function() {
 	Engine.prototype['setOnExecute'] = Engine.prototype.setOnExecute;
 	Engine.prototype['setOnExit'] = Engine.prototype.setOnExit;
 	Engine.prototype['copyToFS'] = Engine.prototype.copyToFS;
+	Engine.prototype['setPersistentPaths'] = Engine.prototype.setPersistentPaths;
 	return Engine;
 })();

platform/javascript/javascript_main.cpp

@@ -88,12 +88,27 @@ void main_loop_callback() {
 		/* clang-format on */
 		os->get_main_loop()->finish();
 		os->finalize_async(); // Will add all the async finish functions.
+		/* clang-format off */
 		EM_ASM({
 			Promise.all(Module.async_finish).then(function() {
 				Module.async_finish = [];
+				return new Promise(function(accept, reject) {
+					if (!Module.idbfs) {
+						accept();
+						return;
+					}
+					FS.syncfs(function(error) {
+						if (error) {
+							err('Failed to save IDB file system: ' + error.message);
+						}
+						accept();
+					});
+				});
+			}).then(function() {
 				ccall("cleanup_after_sync", null, []);
 			});
 		});
+		/* clang-format on */
 	}
 }
 
@@ -101,37 +116,8 @@ extern "C" EMSCRIPTEN_KEEPALIVE void cleanup_after_sync() {
 	emscripten_set_main_loop(exit_callback, -1, false);
 }
 
-extern "C" EMSCRIPTEN_KEEPALIVE void main_after_fs_sync(char *p_idbfs_err) {
-	String idbfs_err = String::utf8(p_idbfs_err);
-	if (!idbfs_err.empty()) {
-		print_line("IndexedDB not available: " + idbfs_err);
-	}
-	os->set_idb_available(idbfs_err.empty());
-	// TODO: Check error return value.
-	Main::setup2(); // Manual second phase.
-	// Ease up compatibility.
-	ResourceLoader::set_abort_on_missing_resources(false);
-	Main::start();
-	os->get_main_loop()->init();
-	// Expose method for requesting quit.
-	EM_ASM({
-		Module['request_quit'] = function() {
-			ccall("_request_quit_callback", null, []);
-		};
-	});
-	// Immediately run the first iteration.
-	// We are inside an animation frame, we want to immediately draw on the newly setup canvas.
-	main_loop_callback();
-	emscripten_resume_main_loop();
-}
-
+/// When calling main, it is assumed FS is setup and synced.
 int main(int argc, char *argv[]) {
-	// Create and mount userfs immediately.
-	EM_ASM({
-		FS.mkdir('/userfs');
-		FS.mount(IDBFS, {}, '/userfs');
-	});
-
 	// Configure locale.
 	char locale_ptr[16];
 	/* clang-format off */
@@ -149,26 +135,30 @@ int main(int argc, char *argv[]) {
 	/* clang-format on */
 
 	os = new OS_JavaScript();
+	os->set_idb_available((bool)EM_ASM_INT({ return Module.idbfs }));
 
 	// We must override main when testing is enabled
 	TEST_MAIN_OVERRIDE
 
-	Main::setup(argv[0], argc - 1, &argv[1], false);
-	emscripten_set_main_loop(main_loop_callback, -1, false);
-	emscripten_pause_main_loop(); // Will need to wait for FS sync.
+	Main::setup(argv[0], argc - 1, &argv[1]);
 
-	// Sync from persistent state into memory and then
-	// run the 'main_after_fs_sync' function.
+	// Ease up compatibility.
+	ResourceLoader::set_abort_on_missing_resources(false);
+
+	Main::start();
+	os->get_main_loop()->init();
+	// Expose method for requesting quit.
 	/* clang-format off */
 	EM_ASM({
-		FS.syncfs(true, function(err) {
-			requestAnimationFrame(function() {
-				ccall('main_after_fs_sync', null, ['string'], [err ? err.message : ""]);
-			});
-		});
+		Module['request_quit'] = function() {
+			ccall("_request_quit_callback", null, []);
+		};
 	});
 	/* clang-format on */
+	emscripten_set_main_loop(main_loop_callback, -1, false);
+	// Immediately run the first iteration.
+	// We are inside an animation frame, we want to immediately draw on the newly setup canvas.
+	main_loop_callback();
 
 	return 0;
-	// Continued async in main_after_fs_sync() from the syncfs() callback.
 }

platform/javascript/native/utils.js

@@ -28,6 +28,38 @@
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
 /*************************************************************************/
 
+Module['initFS'] = function(persistentPaths) {
+	FS.mkdir('/userfs');
+	FS.mount(IDBFS, {}, '/userfs');
+
+	function createRecursive(dir) {
+		try {
+			FS.stat(dir);
+		} catch (e) {
+			if (e.errno !== ERRNO_CODES.ENOENT) {
+				throw e;
+			}
+			FS.mkdirTree(dir);
+		}
+	}
+
+	persistentPaths.forEach(function(path) {
+		createRecursive(path);
+		FS.mount(IDBFS, {}, path);
+	});
+	return new Promise(function(resolve, reject) {
+		FS.syncfs(true, function(err) {
+			if (err) {
+				Module.idbfs = false;
+				console.log("IndexedDB not available: " + err.message);
+			} else {
+				Module.idbfs = true;
+			}
+			resolve(err);
+		});
+	});
+};
+
 Module['copyToFS'] = function(path, buffer) {
 	var p = path.lastIndexOf("/");
 	var dir = "/";
@@ -37,7 +69,7 @@ Module['copyToFS'] = function(path, buffer) {
 	try {
 		FS.stat(dir);
 	} catch (e) {
-		if (e.errno !== ERRNO_CODES.ENOENT) { // 'ENOENT', see https://github.com/emscripten-core/emscripten/blob/master/system/lib/libc/musl/arch/emscripten/bits/errno.h
+		if (e.errno !== ERRNO_CODES.ENOENT) {
 			throw e;
 		}
 		FS.mkdirTree(dir);

platform/javascript/os_javascript.cpp

@@ -72,27 +72,24 @@ MainLoop *OS_JavaScript::get_main_loop() const {
 	return main_loop;
 }
 
-void OS_JavaScript::main_loop_callback() {
-	get_singleton()->main_loop_iterate();
+extern "C" EMSCRIPTEN_KEEPALIVE void _idb_synced() {
+	OS_JavaScript::get_singleton()->idb_is_syncing = false;
 }
 
 bool OS_JavaScript::main_loop_iterate() {
-	if (is_userfs_persistent() && sync_wait_time >= 0) {
-		int64_t current_time = get_ticks_msec();
-		int64_t elapsed_time = current_time - last_sync_check_time;
-		last_sync_check_time = current_time;
-
-		sync_wait_time -= elapsed_time;
-
-		if (sync_wait_time < 0) {
-			/* clang-format off */
-			EM_ASM(
-				FS.syncfs(function(error) {
-					if (error) { err('Failed to save IDB file system: ' + error.message); }
-				});
-			);
-			/* clang-format on */
-		}
+	if (is_userfs_persistent() && idb_needs_sync && !idb_is_syncing) {
+		idb_is_syncing = true;
+		idb_needs_sync = false;
+		/* clang-format off */
+		EM_ASM({
+			FS.syncfs(function(error) {
+				if (error) {
+					err('Failed to save IDB file system: ' + error.message);
+				}
+				ccall("_idb_synced", 'void', [], []);
+			});
+		});
+		/* clang-format on */
 	}
 
 	DisplayServer::get_singleton()->process_events();
@@ -200,11 +197,17 @@ String OS_JavaScript::get_data_path() const {
 }
 
 void OS_JavaScript::file_access_close_callback(const String &p_file, int p_flags) {
-	OS_JavaScript *os = get_singleton();
-	if (os->is_userfs_persistent() && p_file.begins_with("/userfs") && p_flags & FileAccess::WRITE) {
-		os->last_sync_check_time = OS::get_singleton()->get_ticks_msec();
-		// Wait five seconds in case more files are about to be closed.
-		os->sync_wait_time = 5000;
+	OS_JavaScript *os = OS_JavaScript::get_singleton();
+	if (!(os->is_userfs_persistent() && (p_flags & FileAccess::WRITE))) {
+		return; // FS persistence is not working or we are not writing.
+	}
+	bool is_file_persistent = p_file.begins_with("/userfs");
+#ifdef TOOLS_ENABLED
+	// Hack for editor persistence (can we track).
+	is_file_persistent = is_file_persistent || p_file.begins_with("/home/web_user/");
+#endif
+	if (is_file_persistent) {
+		os->idb_needs_sync = true;
 	}
 }
 

platform/javascript/os_javascript.h

@@ -44,8 +44,7 @@ class OS_JavaScript : public OS_Unix {
 
 	bool finalizing = false;
 	bool idb_available = false;
-	int64_t sync_wait_time = -1;
-	int64_t last_sync_check_time = -1;
+	bool idb_needs_sync = false;
 
 	static void main_loop_callback();
 
@@ -62,6 +61,8 @@ protected:
 	bool _check_internal_feature_support(const String &p_feature) override;
 
 public:
+	bool idb_is_syncing = false;
+
 	// Override return type to make writing static callbacks less tedious.
 	static OS_JavaScript *get_singleton();