Merge pull request #39040 from Xrayez/modules-search-path-3.2

[3.2] Add `custom_modules` build option to compile external user modules

SConstruct

@@ -48,8 +48,6 @@ for x in sorted(glob.glob("platform/*")):
     sys.path.remove(tmppath)
     sys.modules.pop('detect')
 
-module_list = methods.detect_modules()
-
 methods.save_active_platforms(active_platforms, active_platform_ids)
 
 custom_tools = ['default']
@@ -121,6 +119,7 @@ opts.Add(BoolVariable('deprecated', "Enable deprecated features", True))
 opts.Add(BoolVariable('gdscript', "Enable GDScript support", True))
 opts.Add(BoolVariable('minizip', "Enable ZIP archive support using minizip", True))
 opts.Add(BoolVariable('xaudio2', "Enable the XAudio2 audio driver", False))
+opts.Add("custom_modules", "A list of comma-separated directory paths containing custom modules to build.", "")
 
 # Advanced options
 opts.Add(BoolVariable('verbose', "Enable verbose output for the compilation", False))
@@ -177,17 +176,41 @@ for k in platform_opts.keys():
     for o in opt_list:
         opts.Add(o)
 
-for x in module_list:
-    module_enabled = True
-    tmppath = "./modules/" + x
-    sys.path.insert(0, tmppath)
+# Detect modules.
+modules_detected = {}
+module_search_paths = ["modules"]  # Built-in path.
+
+if ARGUMENTS.get("custom_modules"):
+    paths = ARGUMENTS.get("custom_modules").split(",")
+    for p in paths:
+        try:
+            module_search_paths.append(methods.convert_custom_modules_path(p))
+        except ValueError as e:
+            print(e)
+            sys.exit(255)
+
+for path in module_search_paths:
+    # Note: custom modules can override built-in ones.
+    modules_detected.update(methods.detect_modules(path))
+    include_path = os.path.dirname(path)
+    if include_path:
+        env_base.Prepend(CPPPATH=[include_path])
+
+# Add module options
+for name, path in modules_detected.items():
+    enabled = True
+    sys.path.insert(0, path)
     import config
-    enabled_attr = getattr(config, "is_enabled", None)
-    if (callable(enabled_attr) and not config.is_enabled()):
-        module_enabled = False
-    sys.path.remove(tmppath)
-    sys.modules.pop('config')
-    opts.Add(BoolVariable('module_' + x + '_enabled', "Enable module '%s'" % (x, ), module_enabled))
+
+    try:
+        enabled = config.is_enabled()
+    except AttributeError:
+        pass
+    sys.path.remove(path)
+    sys.modules.pop("config")
+    opts.Add(BoolVariable("module_" + name + "_enabled", "Enable module '%s'" % (name,), enabled))
+
+methods.write_modules(modules_detected)
 
 opts.Update(env_base)  # update environment
 Help(opts.GenerateHelpText(env_base))  # generate help
@@ -427,16 +450,15 @@ if selected_platform in platform_list:
     sys.path.remove(tmppath)
     sys.modules.pop('detect')
 
-    env.module_list = []
+    modules_enabled = {}
     env.module_icons_paths = []
     env.doc_class_path = {}
 
-    for x in module_list:
-        if not env['module_' + x + '_enabled']:
+    for name, path in sorted(modules_detected.items()):
+        if not env["module_" + name + "_enabled"]:
             continue
-        tmppath = "./modules/" + x
-        sys.path.insert(0, tmppath)
-        env.current_module = x
+        sys.path.insert(0, path)
+        env.current_module = name
         import config
         # can_build changed number of arguments between 3.0 (1) and 3.1 (2),
         # so try both to preserve compatibility for 3.0 modules
@@ -450,26 +472,27 @@ if selected_platform in platform_list:
             can_build = config.can_build(selected_platform)
         if (can_build):
             config.configure(env)
-            env.module_list.append(x)
-
             # Get doc classes paths (if present)
             try:
                 doc_classes = config.get_doc_classes()
                 doc_path = config.get_doc_path()
                 for c in doc_classes:
-                    env.doc_class_path[c] = "modules/" + x + "/" + doc_path
+                    env.doc_class_path[c] = path + "/" + doc_path
             except:
                 pass
             # Get icon paths (if present)
             try:
                 icons_path = config.get_icons_path()
-                env.module_icons_paths.append("modules/" + x + "/" + icons_path)
+                env.module_icons_paths.append(path + "/" + icons_path)
             except:
                 # Default path for module icons
-                env.module_icons_paths.append("modules/" + x + "/" + "icons")
+                env.module_icons_paths.append(path + "/" + "icons")
+            modules_enabled[name] = path
+
+        sys.path.remove(path)
+        sys.modules.pop("config")
 
-        sys.path.remove(tmppath)
-        sys.modules.pop('config')
+    env.module_list = modules_enabled
 
     methods.update_version(env.module_version_string)
 

editor/SCsub

@@ -6,6 +6,7 @@ env.editor_sources = []
 
 import os
 import os.path
+import glob
 from platform_methods import run_in_subprocess
 from compat import open_utf8
 import editor_builders
@@ -41,20 +42,21 @@ if env['tools']:
         f.write(reg_exporters_inc)
         f.write(reg_exporters)
 
-    # API documentation
+    # Core API documentation.
     docs = []
-    doc_dirs = ["doc/classes"]
+    docs += Glob("#doc/classes/*.xml")
 
-    for p in env.doc_class_path.values():
-        if p not in doc_dirs:
-            doc_dirs.append(p)
+    # Module API documentation.
+    module_dirs = []
+    for d in env.doc_class_path.values():
+        if d not in module_dirs:
+            module_dirs.append(d)
 
-    for d in doc_dirs:
-        try:
-            for f in os.listdir(os.path.join(env.Dir('#').abspath, d)):
-                docs.append("#" + os.path.join(d, f))
-        except OSError:
-            pass
+    for d in module_dirs:
+        if not os.path.isabs(d):
+            docs += Glob("#" + d + "/*.xml")  # Built-in.
+        else:
+            docs += Glob(d + "/*.xml")  # Custom.
 
     _make_doc_data_class_path(os.path.join(env.Dir('#').abspath, "editor/doc"))
 
@@ -62,9 +64,7 @@ if env['tools']:
     env.Depends("#editor/doc_data_compressed.gen.h", docs)
     env.CommandNoCache("#editor/doc_data_compressed.gen.h", docs, run_in_subprocess(editor_builders.make_doc_header))
 
-    import glob
-
-    path = env.Dir('.').abspath
+    path = env.Dir(".").abspath
 
     # Translations
     tlist = glob.glob(path + "/translations/*.po")

editor/icons/SCsub

@@ -2,6 +2,8 @@
 
 Import('env')
 
+import os
+
 from platform_methods import run_in_subprocess
 import editor_icons_builders
 
@@ -15,7 +17,10 @@ env['BUILDERS']['MakeEditorIconsBuilder'] = make_editor_icons_builder
 icon_sources = Glob("*.svg")
 
 # Module icons
-for module_icons in env.module_icons_paths:
-    icon_sources += Glob('#' + module_icons + "/*.svg")
+for path in env.module_icons_paths:
+    if not os.path.isabs(path):
+        icon_sources += Glob("#" + path + "/*.svg")  # Built-in.
+    else:
+        icon_sources += Glob(path + "/*.svg")  # Custom.
 
 env.Alias('editor_icons', [env.MakeEditorIconsBuilder('#editor/editor_icons.gen.h', icon_sources)])

main/main.cpp

@@ -1554,7 +1554,11 @@ bool Main::start() {
 		print_line("Loading docs...");
 
 		for (int i = 0; i < _doc_data_class_path_count; i++) {
-			String path = doc_tool.plus_file(_doc_data_class_paths[i].path);
+			// Custom modules are always located by absolute path.
+			String path = _doc_data_class_paths[i].path;
+			if (path.is_rel_path()) {
+				path = doc_tool.plus_file(path);
+			}
 			String name = _doc_data_class_paths[i].name;
 			doc_data_classes[name] = path;
 			if (!checked_paths.has(path)) {

methods.py

@@ -130,31 +130,41 @@ def parse_cg_file(fname, uniforms, sizes, conditionals):
     fs.close()
 
 
-def detect_modules():
+def detect_modules(at_path):
+    module_list = {}  # name : path
 
-    module_list = []
+    modules_glob = os.path.join(at_path, "*")
+    files = glob.glob(modules_glob)
+    files.sort()  # so register_module_types does not change that often, and also plugins are registered in alphabetic order
+
+    for x in files:
+        if not is_module(x):
+            continue
+        name = os.path.basename(x)
+        path = x.replace("\\", "/")  # win32
+        module_list[name] = path
+
+    return module_list
+
+
+def is_module(path):
+    return os.path.isdir(path) and os.path.exists(path + "/config.py")
+
+
+def write_modules(module_list):
     includes_cpp = ""
     register_cpp = ""
     unregister_cpp = ""
 
-    files = glob.glob("modules/*")
-    files.sort()  # so register_module_types does not change that often, and also plugins are registered in alphabetic order
-    for x in files:
-        if not os.path.isdir(x):
-            continue
-        if not os.path.exists(x + "/config.py"):
-            continue
-        x = x.replace("modules/", "")  # rest of world
-        x = x.replace("modules\\", "")  # win32
-        module_list.append(x)
+    for name, path in module_list.items():
         try:
-            with open("modules/" + x + "/register_types.h"):
-                includes_cpp += '#include "modules/' + x + '/register_types.h"\n'
-                register_cpp += '#ifdef MODULE_' + x.upper() + '_ENABLED\n'
-                register_cpp += '\tregister_' + x + '_types();\n'
+            with open(os.path.join(path, "register_types.h")):
+                includes_cpp += '#include "' + path + '/register_types.h"\n'
+                register_cpp += '#ifdef MODULE_' + name.upper() + '_ENABLED\n'
+                register_cpp += '\tregister_' + name + '_types();\n'
                 register_cpp += '#endif\n'
-                unregister_cpp += '#ifdef MODULE_' + x.upper() + '_ENABLED\n'
-                unregister_cpp += '\tunregister_' + x + '_types();\n'
+                unregister_cpp += '#ifdef MODULE_' + name.upper() + '_ENABLED\n'
+                unregister_cpp += '\tunregister_' + name + '_types();\n'
                 unregister_cpp += '#endif\n'
         except IOError:
             pass
@@ -178,7 +188,18 @@ void unregister_module_types() {
     with open("modules/register_module_types.gen.cpp", "w") as f:
         f.write(modules_cpp)
 
-    return module_list
+
+def convert_custom_modules_path(path):
+    if not path:
+        return path
+    err_msg = "Build option 'custom_modules' must %s"
+    if not os.path.isdir(path):
+        raise ValueError(err_msg % "point to an existing directory.")
+    if os.path.realpath(path) == os.path.realpath("modules"):
+        raise ValueError(err_msg % "be a directory other than built-in `modules` directory.")
+    if is_module(path):
+        raise ValueError(err_msg % "point to a directory with modules, not a single module.")
+    return os.path.realpath(os.path.expanduser(path))
 
 
 def disable_module(self):

modules/SCsub

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 
-Import('env')
+Import("env")
+
+import os
 
 env_modules = env.Clone()
 
@@ -10,11 +12,15 @@ env.modules_sources = []
 
 env_modules.add_source_files(env.modules_sources, "register_module_types.gen.cpp")
 
-for x in env.module_list:
-    if (x in env.disabled_modules):
+for name, path in env.module_list.items():
+    if (name in env.disabled_modules):
         continue
-    env_modules.Append(CPPDEFINES=["MODULE_" + x.upper() + "_ENABLED"])
-    SConscript(x + "/SCsub")
+
+    env_modules.Append(CPPDEFINES=["MODULE_" + name.upper() + "_ENABLED"])
+    if not os.path.isabs(path):
+        SConscript(name + "/SCsub")  # Built-in.
+    else:
+        SConscript(path + "/SCsub")  # Custom.
 
 if env['split_libmodules']:
     env.split_lib("modules", env_lib = env_modules)