Merge pull request #36922 from Xrayez/modules-search-path

Add `custom_modules` build option to compile external, user-defined C++ modules

SConstruct

@@ -50,8 +50,6 @@ for x in sorted(glob.glob("platform/*")):
     sys.path.remove(tmppath)
     sys.modules.pop("detect")
 
-module_list = methods.detect_modules()
-
 methods.save_active_platforms(active_platforms, active_platform_ids)
 
 custom_tools = ["default"]
@@ -123,6 +121,7 @@ opts.Add(BoolVariable("use_precise_math_checks", "Math checks use very precise e
 opts.Add(BoolVariable("deprecated", "Enable deprecated features", True))
 opts.Add(BoolVariable("minizip", "Enable ZIP archive support using minizip", True))
 opts.Add(BoolVariable("xaudio2", "Enable the XAudio2 audio driver", False))
+opts.Add("custom_modules", "A list of comma-separated directory paths containing custom modules to build.", "")
 
 # Advanced options
 opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
@@ -181,18 +180,41 @@ for k in platform_opts.keys():
     for o in opt_list:
         opts.Add(o)
 
-for x in module_list:
-    module_enabled = True
-    tmppath = "./modules/" + x
-    sys.path.insert(0, tmppath)
+# Detect modules.
+modules_detected = {}
+module_search_paths = ["modules"]  # Built-in path.
+
+if ARGUMENTS.get("custom_modules"):
+    paths = ARGUMENTS.get("custom_modules").split(",")
+    for p in paths:
+        try:
+            module_search_paths.append(methods.convert_custom_modules_path(p))
+        except ValueError as e:
+            print(e)
+            sys.exit(255)
+
+for path in module_search_paths:
+    # Note: custom modules can override built-in ones.
+    modules_detected.update(methods.detect_modules(path))
+    include_path = os.path.dirname(path)
+    if include_path:
+        env_base.Prepend(CPPPATH=[include_path])
+
+# Add module options.
+for name, path in modules_detected.items():
+    enabled = True
+    sys.path.insert(0, path)
     import config
 
-    enabled_attr = getattr(config, "is_enabled", None)
-    if callable(enabled_attr) and not config.is_enabled():
-        module_enabled = False
-    sys.path.remove(tmppath)
+    try:
+        enabled = config.is_enabled()
+    except AttributeError:
+        pass
+    sys.path.remove(path)
     sys.modules.pop("config")
-    opts.Add(BoolVariable("module_" + x + "_enabled", "Enable module '%s'" % (x,), module_enabled))
+    opts.Add(BoolVariable("module_" + name + "_enabled", "Enable module '%s'" % (name,), enabled))
+
+methods.write_modules(modules_detected)
 
 opts.Update(env_base)  # update environment
 Help(opts.GenerateHelpText(env_base))  # generate help
@@ -501,41 +523,41 @@ if selected_platform in platform_list:
     sys.path.remove(tmppath)
     sys.modules.pop("detect")
 
-    env.module_list = []
+    modules_enabled = {}
     env.module_icons_paths = []
     env.doc_class_path = {}
 
-    for x in sorted(module_list):
-        if not env["module_" + x + "_enabled"]:
+    for name, path in sorted(modules_detected.items()):
+        if not env["module_" + name + "_enabled"]:
             continue
-        tmppath = "./modules/" + x
-        sys.path.insert(0, tmppath)
-        env.current_module = x
+        sys.path.insert(0, path)
+        env.current_module = name
         import config
 
         if config.can_build(env, selected_platform):
             config.configure(env)
-            env.module_list.append(x)
-
             # Get doc classes paths (if present)
             try:
                 doc_classes = config.get_doc_classes()
                 doc_path = config.get_doc_path()
                 for c in doc_classes:
-                    env.doc_class_path[c] = "modules/" + x + "/" + doc_path
+                    env.doc_class_path[c] = path + "/" + doc_path
             except:
                 pass
             # Get icon paths (if present)
             try:
                 icons_path = config.get_icons_path()
-                env.module_icons_paths.append("modules/" + x + "/" + icons_path)
+                env.module_icons_paths.append(path + "/" + icons_path)
             except:
                 # Default path for module icons
-                env.module_icons_paths.append("modules/" + x + "/" + "icons")
+                env.module_icons_paths.append(path + "/" + "icons")
+            modules_enabled[name] = path
 
-        sys.path.remove(tmppath)
+        sys.path.remove(path)
         sys.modules.pop("config")
 
+    env.module_list = modules_enabled
+
     methods.update_version(env.module_version_string)
 
     env["PROGSUFFIX"] = suffix + env.module_version_string + env["PROGSUFFIX"]

editor/SCsub

@@ -6,6 +6,7 @@ env.editor_sources = []
 
 import os
 import os.path
+import glob
 from platform_methods import run_in_subprocess
 import editor_builders
 
@@ -40,20 +41,21 @@ if env["tools"]:
         f.write(reg_exporters_inc)
         f.write(reg_exporters)
 
-    # API documentation
+    # Core API documentation.
     docs = []
-    doc_dirs = ["doc/classes"]
+    docs += Glob("#doc/classes/*.xml")
 
-    for p in env.doc_class_path.values():
-        if p not in doc_dirs:
-            doc_dirs.append(p)
+    # Module API documentation.
+    module_dirs = []
+    for d in env.doc_class_path.values():
+        if d not in module_dirs:
+            module_dirs.append(d)
 
-    for d in doc_dirs:
-        try:
-            for f in os.listdir(os.path.join(env.Dir("#").abspath, d)):
-                docs.append("#" + os.path.join(d, f))
-        except OSError:
-            pass
+    for d in module_dirs:
+        if not os.path.isabs(d):
+            docs += Glob("#" + d + "/*.xml")  # Built-in.
+        else:
+            docs += Glob(d + "/*.xml")  # Custom.
 
     _make_doc_data_class_path(os.path.join(env.Dir("#").abspath, "editor"))
 
@@ -61,8 +63,6 @@ if env["tools"]:
     env.Depends("#editor/doc_data_compressed.gen.h", docs)
     env.CommandNoCache("#editor/doc_data_compressed.gen.h", docs, run_in_subprocess(editor_builders.make_doc_header))
 
-    import glob
-
     path = env.Dir(".").abspath
 
     # Editor translations

editor/icons/SCsub

@@ -2,6 +2,8 @@
 
 Import("env")
 
+import os
+
 from platform_methods import run_in_subprocess
 import editor_icons_builders
 
@@ -15,7 +17,10 @@ env["BUILDERS"]["MakeEditorIconsBuilder"] = make_editor_icons_builder
 icon_sources = Glob("*.svg")
 
 # Module icons
-for module_icons in env.module_icons_paths:
-    icon_sources += Glob("#" + module_icons + "/*.svg")
+for path in env.module_icons_paths:
+    if not os.path.isabs(path):
+        icon_sources += Glob("#" + path + "/*.svg")  # Built-in.
+    else:
+        icon_sources += Glob(path + "/*.svg")  # Custom.
 
 env.Alias("editor_icons", [env.MakeEditorIconsBuilder("#editor/editor_icons.gen.h", icon_sources)])

main/main.cpp

@@ -1645,7 +1645,11 @@ bool Main::start() {
 		print_line("Loading docs...");
 
 		for (int i = 0; i < _doc_data_class_path_count; i++) {
-			String path = doc_tool.plus_file(_doc_data_class_paths[i].path);
+			// Custom modules are always located by absolute path.
+			String path = _doc_data_class_paths[i].path;
+			if (path.is_rel_path()) {
+				path = doc_tool.plus_file(path);
+			}
 			String name = _doc_data_class_paths[i].name;
 			doc_data_classes[name] = path;
 			if (!checked_paths.has(path)) {

methods.py

@@ -137,37 +137,47 @@ def parse_cg_file(fname, uniforms, sizes, conditionals):
     fs.close()
 
 
-def detect_modules():
+def detect_modules(at_path):
+    module_list = {}  # name : path
 
-    module_list = []
+    modules_glob = os.path.join(at_path, "*")
+    files = glob.glob(modules_glob)
+    files.sort()  # so register_module_types does not change that often, and also plugins are registered in alphabetic order
+
+    for x in files:
+        if not is_module(x):
+            continue
+        name = os.path.basename(x)
+        path = x.replace("\\", "/")  # win32
+        module_list[name] = path
+
+    return module_list
+
+
+def is_module(path):
+    return os.path.isdir(path) and os.path.exists(path + "/config.py")
+
+
+def write_modules(module_list):
     includes_cpp = ""
+    preregister_cpp = ""
     register_cpp = ""
     unregister_cpp = ""
-    preregister_cpp = ""
 
-    files = glob.glob("modules/*")
-    files.sort()  # so register_module_types does not change that often, and also plugins are registered in alphabetic order
-    for x in files:
-        if not os.path.isdir(x):
-            continue
-        if not os.path.exists(x + "/config.py"):
-            continue
-        x = x.replace("modules/", "")  # rest of world
-        x = x.replace("modules\\", "")  # win32
-        module_list.append(x)
+    for name, path in module_list.items():
         try:
-            with open("modules/" + x + "/register_types.h"):
-                includes_cpp += '#include "modules/' + x + '/register_types.h"\n'
-                register_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n"
-                register_cpp += "\tregister_" + x + "_types();\n"
-                register_cpp += "#endif\n"
-                preregister_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n"
-                preregister_cpp += "#ifdef MODULE_" + x.upper() + "_HAS_PREREGISTER\n"
-                preregister_cpp += "\tpreregister_" + x + "_types();\n"
+            with open(os.path.join(path, "register_types.h")):
+                includes_cpp += '#include "' + path + '/register_types.h"\n'
+                preregister_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n"
+                preregister_cpp += "#ifdef MODULE_" + name.upper() + "_HAS_PREREGISTER\n"
+                preregister_cpp += "\tpreregister_" + name + "_types();\n"
                 preregister_cpp += "#endif\n"
                 preregister_cpp += "#endif\n"
-                unregister_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n"
-                unregister_cpp += "\tunregister_" + x + "_types();\n"
+                register_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n"
+                register_cpp += "\tregister_" + name + "_types();\n"
+                register_cpp += "#endif\n"
+                unregister_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n"
+                unregister_cpp += "\tunregister_" + name + "_types();\n"
                 unregister_cpp += "#endif\n"
         except IOError:
             pass
@@ -202,7 +212,18 @@ void unregister_module_types() {
     with open("modules/register_module_types.gen.cpp", "w") as f:
         f.write(modules_cpp)
 
-    return module_list
+
+def convert_custom_modules_path(path):
+    if not path:
+        return path
+    err_msg = "Build option 'custom_modules' must %s"
+    if not os.path.isdir(path):
+        raise ValueError(err_msg % "point to an existing directory.")
+    if os.path.realpath(path) == os.path.realpath("modules"):
+        raise ValueError(err_msg % "be a directory other than built-in `modules` directory.")
+    if is_module(path):
+        raise ValueError(err_msg % "point to a directory with modules, not a single module.")
+    return os.path.realpath(os.path.expanduser(path))
 
 
 def disable_module(self):

modules/SCsub

@@ -3,6 +3,7 @@
 Import("env")
 
 import modules_builders
+import os
 
 env_modules = env.Clone()
 
@@ -13,16 +14,20 @@ env.CommandNoCache("modules_enabled.gen.h", Value(env.module_list), modules_buil
 
 vs_sources = []
 # libmodule_<name>.a for each active module.
-for module in env.module_list:
+for name, path in env.module_list.items():
     env.modules_sources = []
-    SConscript(module + "/SCsub")
+
+    if not os.path.isabs(path):
+        SConscript(name + "/SCsub")  # Built-in.
+    else:
+        SConscript(path + "/SCsub")  # Custom.
 
     # Some modules are not linked automatically but can be enabled optionally
     # on iOS, so we handle those specially.
-    if env["platform"] == "iphone" and module in ["arkit", "camera"]:
+    if env["platform"] == "iphone" and name in ["arkit", "camera"]:
         continue
 
-    lib = env_modules.add_library("module_%s" % module, env.modules_sources)
+    lib = env_modules.add_library("module_%s" % name, env.modules_sources)
     env.Prepend(LIBS=[lib])
     if env["vsproj"]:
         vs_sources += env.modules_sources