SCons: Make builders prettier, utilize `constexpr`

SConstruct

@@ -968,8 +968,6 @@ if env.editor_build:
         print_error("Not all modules required by editor builds are enabled.")
         Exit(255)
 
-env.version_info = methods.get_version_info(env.module_version_string)
-
 env["PROGSUFFIX_WRAP"] = suffix + env.module_version_string + ".console" + env["PROGSUFFIX"]
 env["PROGSUFFIX"] = suffix + env.module_version_string + env["PROGSUFFIX"]
 env["OBJSUFFIX"] = suffix + env["OBJSUFFIX"]

core/SCsub

@@ -167,10 +167,9 @@ env.add_source_files(env.core_sources, "*.cpp")
 
 # Generate disabled classes
 def disabled_class_builder(target, source, env):
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         for c in source[0].read():
-            cs = c.strip()
-            if cs != "":
+            if cs := c.strip():
                 file.write(f"#define ClassDB_Disable_{cs} 1\n")
 
 
@@ -179,7 +178,7 @@ env.CommandNoCache("disabled_classes.gen.h", env.Value(env.disabled_classes), en
 
 # Generate version info
 def version_info_builder(target, source, env):
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             """\
 #define VERSION_SHORT_NAME "{short_name}"
@@ -193,35 +192,37 @@ def version_info_builder(target, source, env):
 #define VERSION_WEBSITE "{website}"
 #define VERSION_DOCS_BRANCH "{docs_branch}"
 #define VERSION_DOCS_URL "https://docs.godotengine.org/en/" VERSION_DOCS_BRANCH
-""".format(**env.version_info)
+""".format(**source[0].read())
         )
 
 
-env.CommandNoCache("version_generated.gen.h", env.Value(env.version_info), env.Run(version_info_builder))
+env.CommandNoCache(
+    "version_generated.gen.h",
+    env.Value(methods.get_version_info(env.module_version_string)),
+    env.Run(version_info_builder),
+)
 
 
 # Generate version hash
 def version_hash_builder(target, source, env):
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             """\
 #include "core/version.h"
 
 const char *const VERSION_HASH = "{git_hash}";
 const uint64_t VERSION_TIMESTAMP = {git_timestamp};
-""".format(**env.version_info)
+""".format(**source[0].read())
         )
 
 
-gen_hash = env.CommandNoCache(
-    "version_hash.gen.cpp", env.Value(env.version_info["git_hash"]), env.Run(version_hash_builder)
-)
+gen_hash = env.CommandNoCache("version_hash.gen.cpp", env.Value(methods.get_git_info()), env.Run(version_hash_builder))
 env.add_source_files(env.core_sources, gen_hash)
 
 
 # Generate AES256 script encryption key
 def encryption_key_builder(target, source, env):
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             f"""\
 #include "core/config/project_settings.h"
@@ -251,30 +252,21 @@ env.add_source_files(env.core_sources, gen_encrypt)
 
 
 # Certificates
-env.Depends(
-    "#core/io/certs_compressed.gen.h",
-    ["#thirdparty/certs/ca-certificates.crt", env.Value(env["builtin_certs"]), env.Value(env["system_certs_path"])],
-)
 env.CommandNoCache(
     "#core/io/certs_compressed.gen.h",
-    "#thirdparty/certs/ca-certificates.crt",
+    ["#thirdparty/certs/ca-certificates.crt", env.Value(env["builtin_certs"]), env.Value(env["system_certs_path"])],
     env.Run(core_builders.make_certs_header),
 )
 
 # Authors
-env.Depends("#core/authors.gen.h", "../AUTHORS.md")
-env.CommandNoCache("#core/authors.gen.h", "../AUTHORS.md", env.Run(core_builders.make_authors_header))
+env.CommandNoCache("#core/authors.gen.h", "#AUTHORS.md", env.Run(core_builders.make_authors_header))
 
 # Donors
-env.Depends("#core/donors.gen.h", "../DONORS.md")
-env.CommandNoCache("#core/donors.gen.h", "../DONORS.md", env.Run(core_builders.make_donors_header))
+env.CommandNoCache("#core/donors.gen.h", "#DONORS.md", env.Run(core_builders.make_donors_header))
 
 # License
-env.Depends("#core/license.gen.h", ["../COPYRIGHT.txt", "../LICENSE.txt"])
 env.CommandNoCache(
-    "#core/license.gen.h",
-    ["../COPYRIGHT.txt", "../LICENSE.txt"],
-    env.Run(core_builders.make_license_header),
+    "#core/license.gen.h", ["#COPYRIGHT.txt", "#LICENSE.txt"], env.Run(core_builders.make_license_header)
 )
 
 # Chain load SCsubs

core/core_builders.py

@@ -1,171 +1,104 @@
 """Functions used to generate source files during build time"""
 
-import zlib
+from collections import OrderedDict
+from io import TextIOWrapper
 
-
-def escape_string(s):
-    def charcode_to_c_escapes(c):
-        rev_result = []
-        while c >= 256:
-            c, low = (c // 256, c % 256)
-            rev_result.append("\\%03o" % low)
-        rev_result.append("\\%03o" % c)
-        return "".join(reversed(rev_result))
-
-    result = ""
-    if isinstance(s, str):
-        s = s.encode("utf-8")
-    for c in s:
-        if not (32 <= c < 127) or c in (ord("\\"), ord('"')):
-            result += charcode_to_c_escapes(c)
-        else:
-            result += chr(c)
-    return result
+import methods
 
 
 def make_certs_header(target, source, env):
-    src = str(source[0])
-    dst = str(target[0])
-    with open(src, "rb") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
-        buf = f.read()
-        decomp_size = len(buf)
-
-        # Use maximum zlib compression level to further reduce file size
-        # (at the cost of initial build times).
-        buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
-
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef CERTS_COMPRESSED_GEN_H\n")
-        g.write("#define CERTS_COMPRESSED_GEN_H\n")
+    buffer = methods.get_buffer(str(source[0]))
+    decomp_size = len(buffer)
+    buffer = methods.compress_buffer(buffer)
 
+    with methods.generated_wrapper(str(target[0])) as file:
         # System certs path. Editor will use them if defined. (for package maintainers)
-        path = env["system_certs_path"]
-        g.write('#define _SYSTEM_CERTS_PATH "%s"\n' % str(path))
+        file.write('#define _SYSTEM_CERTS_PATH "{}"\n'.format(env["system_certs_path"]))
         if env["builtin_certs"]:
             # Defined here and not in env so changing it does not trigger a full rebuild.
-            g.write("#define BUILTIN_CERTS_ENABLED\n")
-            g.write("static const int _certs_compressed_size = " + str(len(buf)) + ";\n")
-            g.write("static const int _certs_uncompressed_size = " + str(decomp_size) + ";\n")
-            g.write("static const unsigned char _certs_compressed[] = {\n")
-            for i in range(len(buf)):
-                g.write("\t" + str(buf[i]) + ",\n")
-            g.write("};\n")
-        g.write("#endif // CERTS_COMPRESSED_GEN_H")
+            file.write(f"""\
+#define BUILTIN_CERTS_ENABLED
+
+inline constexpr int _certs_compressed_size = {len(buffer)};
+inline constexpr int _certs_uncompressed_size = {decomp_size};
+inline constexpr unsigned char _certs_compressed[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")
 
 
 def make_authors_header(target, source, env):
-    sections = [
-        "Project Founders",
-        "Lead Developer",
-        "Project Manager",
-        "Developers",
-    ]
-    sections_id = [
-        "AUTHORS_FOUNDERS",
-        "AUTHORS_LEAD_DEVELOPERS",
-        "AUTHORS_PROJECT_MANAGERS",
-        "AUTHORS_DEVELOPERS",
-    ]
-
-    src = str(source[0])
-    dst = str(target[0])
-    with open(src, "r", encoding="utf-8") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef AUTHORS_GEN_H\n")
-        g.write("#define AUTHORS_GEN_H\n")
-
-        reading = False
+    SECTIONS = {
+        "Project Founders": "AUTHORS_FOUNDERS",
+        "Lead Developer": "AUTHORS_LEAD_DEVELOPERS",
+        "Project Manager": "AUTHORS_PROJECT_MANAGERS",
+        "Developers": "AUTHORS_DEVELOPERS",
+    }
+    buffer = methods.get_buffer(str(source[0]))
+    reading = False
+
+    with methods.generated_wrapper(str(target[0])) as file:
 
         def close_section():
-            g.write("\t0\n")
-            g.write("};\n")
-
-        for line in f:
-            if reading:
-                if line.startswith("    "):
-                    g.write('\t"' + escape_string(line.strip()) + '",\n')
-                    continue
-            if line.startswith("## "):
+            file.write("\tnullptr,\n};\n\n")
+
+        for line in buffer.decode().splitlines():
+            if line.startswith("    ") and reading:
+                file.write(f'\t"{methods.to_escaped_cstring(line).strip()}",\n')
+            elif line.startswith("## "):
                 if reading:
                     close_section()
                     reading = False
-                for section, section_id in zip(sections, sections_id):
-                    if line.strip().endswith(section):
-                        current_section = escape_string(section_id)
-                        reading = True
-                        g.write("const char *const " + current_section + "[] = {\n")
-                        break
+                section = SECTIONS[line[3:].strip()]
+                if section:
+                    file.write(f"inline constexpr const char *{section}[] = {{\n")
+                    reading = True
 
         if reading:
             close_section()
 
-        g.write("#endif // AUTHORS_GEN_H\n")
-
 
 def make_donors_header(target, source, env):
-    sections = [
-        "Patrons",
-        "Platinum sponsors",
-        "Gold sponsors",
-        "Silver sponsors",
-        "Diamond members",
-        "Titanium members",
-        "Platinum members",
-        "Gold members",
-    ]
-    sections_id = [
-        "DONORS_PATRONS",
-        "DONORS_SPONSORS_PLATINUM",
-        "DONORS_SPONSORS_GOLD",
-        "DONORS_SPONSORS_SILVER",
-        "DONORS_MEMBERS_DIAMOND",
-        "DONORS_MEMBERS_TITANIUM",
-        "DONORS_MEMBERS_PLATINUM",
-        "DONORS_MEMBERS_GOLD",
-    ]
-
-    src = str(source[0])
-    dst = str(target[0])
-    with open(src, "r", encoding="utf-8") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef DONORS_GEN_H\n")
-        g.write("#define DONORS_GEN_H\n")
-
-        reading = False
+    SECTIONS = {
+        "Patrons": "DONORS_PATRONS",
+        "Platinum sponsors": "DONORS_SPONSORS_PLATINUM",
+        "Gold sponsors": "DONORS_SPONSORS_GOLD",
+        "Silver sponsors": "DONORS_SPONSORS_SILVER",
+        "Diamond members": "DONORS_MEMBERS_DIAMOND",
+        "Titanium members": "DONORS_MEMBERS_TITANIUM",
+        "Platinum members": "DONORS_MEMBERS_PLATINUM",
+        "Gold members": "DONORS_MEMBERS_GOLD",
+    }
+    buffer = methods.get_buffer(str(source[0]))
+    reading = False
+
+    with methods.generated_wrapper(str(target[0])) as file:
 
         def close_section():
-            g.write("\t0\n")
-            g.write("};\n")
-
-        for line in f:
-            if reading >= 0:
-                if line.startswith("    "):
-                    g.write('\t"' + escape_string(line.strip()) + '",\n')
-                    continue
-            if line.startswith("## "):
+            file.write("\tnullptr,\n};\n\n")
+
+        for line in buffer.decode().splitlines():
+            if line.startswith("    ") and reading:
+                file.write(f'\t"{methods.to_escaped_cstring(line).strip()}",\n')
+            elif line.startswith("## "):
                 if reading:
                     close_section()
                     reading = False
-                for section, section_id in zip(sections, sections_id):
-                    if line.strip().endswith(section):
-                        current_section = escape_string(section_id)
-                        reading = True
-                        g.write("const char *const " + current_section + "[] = {\n")
-                        break
+                section = SECTIONS.get(line[3:].strip())
+                if section:
+                    file.write(f"inline constexpr const char *{section}[] = {{\n")
+                    reading = True
 
         if reading:
             close_section()
 
-        g.write("#endif // DONORS_GEN_H\n")
-
 
 def make_license_header(target, source, env):
     src_copyright = str(source[0])
     src_license = str(source[1])
-    dst = str(target[0])
 
     class LicenseReader:
-        def __init__(self, license_file):
+        def __init__(self, license_file: TextIOWrapper):
             self._license_file = license_file
             self.line_num = 0
             self.current = self.next_line()
@@ -188,9 +121,7 @@ def make_license_header(target, source, env):
                 lines.append(self.current.strip())
             return (tag, lines)
 
-    from collections import OrderedDict
-
-    projects: dict = OrderedDict()
+    projects = OrderedDict()
     license_list = []
 
     with open(src_copyright, "r", encoding="utf-8") as copyright_file:
@@ -212,7 +143,7 @@ def make_license_header(target, source, env):
                 part = {}
                 reader.next_line()
 
-    data_list: list = []
+    data_list = []
     for project in iter(projects.values()):
         for part in project:
             part["file_index"] = len(data_list)
@@ -220,96 +151,76 @@ def make_license_header(target, source, env):
             part["copyright_index"] = len(data_list)
             data_list += part["Copyright"]
 
-    with open(dst, "w", encoding="utf-8", newline="\n") as f:
-        f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        f.write("#ifndef LICENSE_GEN_H\n")
-        f.write("#define LICENSE_GEN_H\n")
-        f.write("const char *const GODOT_LICENSE_TEXT =")
-
-        with open(src_license, "r", encoding="utf-8") as license_file:
-            for line in license_file:
-                escaped_string = escape_string(line.strip())
-                f.write('\n\t\t"' + escaped_string + '\\n"')
-        f.write(";\n\n")
-
-        f.write(
-            "struct ComponentCopyrightPart {\n"
-            "\tconst char *license;\n"
-            "\tconst char *const *files;\n"
-            "\tconst char *const *copyright_statements;\n"
-            "\tint file_count;\n"
-            "\tint copyright_count;\n"
-            "};\n\n"
-        )
-
-        f.write(
-            "struct ComponentCopyright {\n"
-            "\tconst char *name;\n"
-            "\tconst ComponentCopyrightPart *parts;\n"
-            "\tint part_count;\n"
-            "};\n\n"
-        )
-
-        f.write("const char *const COPYRIGHT_INFO_DATA[] = {\n")
+    with open(src_license, "r", encoding="utf-8") as file:
+        license_text = file.read()
+
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+inline constexpr const char *GODOT_LICENSE_TEXT = {{
+{methods.to_raw_cstring(license_text)}
+}};
+
+struct ComponentCopyrightPart {{
+	const char *license;
+	const char *const *files;
+	const char *const *copyright_statements;
+	int file_count;
+	int copyright_count;
+}};
+
+struct ComponentCopyright {{
+	const char *name;
+	const ComponentCopyrightPart *parts;
+	int part_count;
+}};
+
+""")
+
+        file.write("inline constexpr const char *COPYRIGHT_INFO_DATA[] = {\n")
         for line in data_list:
-            f.write('\t"' + escape_string(line) + '",\n')
-        f.write("};\n\n")
+            file.write(f'\t"{methods.to_escaped_cstring(line)}",\n')
+        file.write("};\n\n")
 
-        f.write("const ComponentCopyrightPart COPYRIGHT_PROJECT_PARTS[] = {\n")
+        file.write("inline constexpr ComponentCopyrightPart COPYRIGHT_PROJECT_PARTS[] = {\n")
         part_index = 0
         part_indexes = {}
         for project_name, project in iter(projects.items()):
             part_indexes[project_name] = part_index
             for part in project:
-                f.write(
-                    '\t{ "'
-                    + escape_string(part["License"][0])
-                    + '", '
-                    + "&COPYRIGHT_INFO_DATA["
-                    + str(part["file_index"])
-                    + "], "
-                    + "&COPYRIGHT_INFO_DATA["
-                    + str(part["copyright_index"])
-                    + "], "
-                    + str(len(part["Files"]))
-                    + ", "
-                    + str(len(part["Copyright"]))
-                    + " },\n"
+                file.write(
+                    f'\t{{ "{methods.to_escaped_cstring(part["License"][0])}", '
+                    + f"&COPYRIGHT_INFO_DATA[{part['file_index']}], "
+                    + f"&COPYRIGHT_INFO_DATA[{part['copyright_index']}], "
+                    + f"{len(part['Files'])}, {len(part['Copyright'])} }},\n"
                 )
                 part_index += 1
-        f.write("};\n\n")
+        file.write("};\n\n")
 
-        f.write("const int COPYRIGHT_INFO_COUNT = " + str(len(projects)) + ";\n")
+        file.write(f"inline constexpr int COPYRIGHT_INFO_COUNT = {len(projects)};\n")
 
-        f.write("const ComponentCopyright COPYRIGHT_INFO[] = {\n")
+        file.write("inline constexpr ComponentCopyright COPYRIGHT_INFO[] = {\n")
         for project_name, project in iter(projects.items()):
-            f.write(
-                '\t{ "'
-                + escape_string(project_name)
-                + '", '
-                + "&COPYRIGHT_PROJECT_PARTS["
-                + str(part_indexes[project_name])
-                + "], "
-                + str(len(project))
-                + " },\n"
+            file.write(
+                f'\t{{ "{methods.to_escaped_cstring(project_name)}", '
+                + f"&COPYRIGHT_PROJECT_PARTS[{part_indexes[project_name]}], "
+                + f"{len(project)} }},\n"
             )
-        f.write("};\n\n")
+        file.write("};\n\n")
 
-        f.write("const int LICENSE_COUNT = " + str(len(license_list)) + ";\n")
+        file.write(f"inline constexpr int LICENSE_COUNT = {len(license_list)};\n")
 
-        f.write("const char *const LICENSE_NAMES[] = {\n")
+        file.write("inline constexpr const char *LICENSE_NAMES[] = {\n")
         for license in license_list:
-            f.write('\t"' + escape_string(license[0]) + '",\n')
-        f.write("};\n\n")
+            file.write(f'\t"{methods.to_escaped_cstring(license[0])}",\n')
+        file.write("};\n\n")
 
-        f.write("const char *const LICENSE_BODIES[] = {\n\n")
+        file.write("inline constexpr const char *LICENSE_BODIES[] = {\n\n")
         for license in license_list:
+            to_raw = []
             for line in license[1:]:
                 if line == ".":
-                    f.write('\t"\\n"\n')
+                    to_raw += [""]
                 else:
-                    f.write('\t"' + escape_string(line) + '\\n"\n')
-            f.write('\t"",\n\n')
-        f.write("};\n\n")
-
-        f.write("#endif // LICENSE_GEN_H\n")
+                    to_raw += [line]
+            file.write(f"{methods.to_raw_cstring(to_raw)},\n\n")
+        file.write("};\n\n")

core/extension/make_interface_dumper.py

@@ -1,52 +1,37 @@
-import zlib
+import methods
 
 
 def run(target, source, env):
-    src = str(source[0])
-    dst = str(target[0])
-    with open(src, "rb") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
-        buf = f.read()
-        decomp_size = len(buf)
-
-        # Use maximum zlib compression level to further reduce file size
-        # (at the cost of initial build times).
-        buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
-
-        g.write(
-            """/* THIS FILE IS GENERATED DO NOT EDIT */
-#pragma once
+    buffer = methods.get_buffer(str(source[0]))
+    decomp_size = len(buffer)
+    buffer = methods.compress_buffer(buffer)
 
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
 #ifdef TOOLS_ENABLED
 
 #include "core/io/compression.h"
 #include "core/io/file_access.h"
 #include "core/string/ustring.h"
 
-"""
-        )
-
-        g.write("static const int _gdextension_interface_data_compressed_size = " + str(len(buf)) + ";\n")
-        g.write("static const int _gdextension_interface_data_uncompressed_size = " + str(decomp_size) + ";\n")
-        g.write("static const unsigned char _gdextension_interface_data_compressed[] = {\n")
-        for i in range(len(buf)):
-            g.write("\t" + str(buf[i]) + ",\n")
-        g.write("};\n")
-
-        g.write(
-            """
-class GDExtensionInterfaceDump {
-    public:
-        static void generate_gdextension_interface_file(const String &p_path) {
-            Ref<FileAccess> fa = FileAccess::open(p_path, FileAccess::WRITE);
-            ERR_FAIL_COND_MSG(fa.is_null(), vformat("Cannot open file '%s' for writing.", p_path));
-            Vector<uint8_t> data;
-            data.resize(_gdextension_interface_data_uncompressed_size);
-            int ret = Compression::decompress(data.ptrw(), _gdextension_interface_data_uncompressed_size, _gdextension_interface_data_compressed, _gdextension_interface_data_compressed_size, Compression::MODE_DEFLATE);
-            ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
-            fa->store_buffer(data.ptr(), data.size());
-        };
-};
+inline constexpr int _gdextension_interface_data_compressed_size = {len(buffer)};
+inline constexpr int _gdextension_interface_data_uncompressed_size = {decomp_size};
+inline constexpr unsigned char _gdextension_interface_data_compressed[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+
+class GDExtensionInterfaceDump {{
+	public:
+		static void generate_gdextension_interface_file(const String &p_path) {{
+			Ref<FileAccess> fa = FileAccess::open(p_path, FileAccess::WRITE);
+			ERR_FAIL_COND_MSG(fa.is_null(), vformat("Cannot open file '%s' for writing.", p_path));
+			Vector<uint8_t> data;
+			data.resize(_gdextension_interface_data_uncompressed_size);
+			int ret = Compression::decompress(data.ptrw(), _gdextension_interface_data_uncompressed_size, _gdextension_interface_data_compressed, _gdextension_interface_data_compressed_size, Compression::MODE_DEFLATE);
+			ERR_FAIL_COND_MSG(ret == -1, "Compressed file is corrupt.");
+			fa->store_buffer(data.ptr(), data.size());
+		}};
+}};
 
 #endif // TOOLS_ENABLED
-"""
-        )
+""")

core/input/input_builders.py

@@ -2,18 +2,22 @@
 
 from collections import OrderedDict
 
+import methods
+
 
 def make_default_controller_mappings(target, source, env):
-    dst = str(target[0])
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write('#include "core/typedefs.h"\n')
-        g.write('#include "core/input/default_controller_mappings.h"\n')
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write("""\
+#include "core/input/default_controller_mappings.h"
+
+#include "core/typedefs.h"
+
+""")
 
         # ensure mappings have a consistent order
-        platform_mappings: dict = OrderedDict()
-        for src_path in source:
-            with open(str(src_path), "r", encoding="utf-8") as f:
+        platform_mappings = OrderedDict()
+        for src_path in map(str, source):
+            with open(src_path, "r", encoding="utf-8") as f:
                 # read mapping file and skip header
                 mapping_file_lines = f.readlines()[2:]
 
@@ -32,28 +36,28 @@ def make_default_controller_mappings(target, source, env):
                     line_parts = line.split(",")
                     guid = line_parts[0]
                     if guid in platform_mappings[current_platform]:
-                        g.write(
+                        file.write(
                             "// WARNING: DATABASE {} OVERWROTE PRIOR MAPPING: {} {}\n".format(
                                 src_path, current_platform, platform_mappings[current_platform][guid]
                             )
                         )
                     platform_mappings[current_platform][guid] = line
 
-        platform_variables = {
-            "Linux": "#ifdef LINUXBSD_ENABLED",
-            "Windows": "#ifdef WINDOWS_ENABLED",
-            "Mac OS X": "#ifdef MACOS_ENABLED",
-            "Android": "#ifdef ANDROID_ENABLED",
-            "iOS": "#ifdef IOS_ENABLED",
-            "Web": "#ifdef WEB_ENABLED",
+        PLATFORM_VARIABLES = {
+            "Linux": "LINUXBSD",
+            "Windows": "WINDOWS",
+            "Mac OS X": "MACOS",
+            "Android": "ANDROID",
+            "iOS": "IOS",
+            "Web": "WEB",
         }
 
-        g.write("const char* DefaultControllerMappings::mappings[] = {\n")
+        file.write("const char *DefaultControllerMappings::mappings[] = {\n")
         for platform, mappings in platform_mappings.items():
-            variable = platform_variables[platform]
-            g.write("{}\n".format(variable))
+            variable = PLATFORM_VARIABLES[platform]
+            file.write(f"#ifdef {variable}_ENABLED\n")
             for mapping in mappings.values():
-                g.write('\t"{}",\n'.format(mapping))
-            g.write("#endif\n")
+                file.write(f'\t"{mapping}",\n')
+            file.write(f"#endif // {variable}_ENABLED\n")
 
-        g.write("\tnullptr\n};\n")
+        file.write("\tnullptr\n};\n")

editor/SCsub

@@ -5,7 +5,6 @@ Import("env")
 
 env.editor_sources = []
 
-import glob
 import os
 
 import editor_builders
@@ -17,17 +16,16 @@ if env.editor_build:
     def doc_data_class_path_builder(target, source, env):
         paths = dict(sorted(source[0].read().items()))
         data = "\n".join([f'\t{{"{key}", "{value}"}},' for key, value in paths.items()])
-        with methods.generated_wrapper(target) as file:
+        with methods.generated_wrapper(str(target[0])) as file:
             file.write(
                 f"""\
-static const int _doc_data_class_path_count = {len(paths)};
-
 struct _DocDataClassPath {{
 	const char *name;
 	const char *path;
 }};
 
-static const _DocDataClassPath _doc_data_class_paths[{len(env.doc_class_path) + 1}] = {{
+inline constexpr int _doc_data_class_path_count = {len(paths)};
+inline constexpr _DocDataClassPath _doc_data_class_paths[{len(env.doc_class_path) + 1}] = {{
 {data}
 	{{nullptr, nullptr}},
 }};
@@ -42,7 +40,7 @@ static const _DocDataClassPath _doc_data_class_paths[{len(env.doc_class_path) +
         exp_inc = "\n".join([f'#include "platform/{p}/export/export.h"' for p in platforms])
         exp_reg = "\n".join([f"\tregister_{p}_exporter();" for p in platforms])
         exp_type = "\n".join([f"\tregister_{p}_exporter_types();" for p in platforms])
-        with methods.generated_wrapper(target) as file:
+        with methods.generated_wrapper(str(target[0])) as file:
             file.write(
                 f"""\
 #include "register_exporters.h"
@@ -83,7 +81,6 @@ void register_exporter_types() {{
             docs += Glob(d + "/*.xml")  # Custom.
 
     docs = sorted(docs)
-    env.Depends("#editor/doc_data_compressed.gen.h", docs)
     env.CommandNoCache(
         "#editor/doc_data_compressed.gen.h",
         docs,
@@ -97,40 +94,31 @@ void register_exporter_types() {{
     # Generated with `make include-list` for each resource.
 
     # Editor translations
-    tlist = glob.glob(env.Dir("#editor/translations/editor").abspath + "/*.po")
-    env.Depends("#editor/editor_translations.gen.h", tlist)
     env.CommandNoCache(
         "#editor/editor_translations.gen.h",
-        tlist,
-        env.Run(editor_builders.make_editor_translations_header),
+        Glob("#editor/translations/editor/*"),
+        env.Run(editor_builders.make_translations_header),
     )
 
     # Property translations
-    tlist = glob.glob(env.Dir("#editor/translations/properties").abspath + "/*.po")
-    env.Depends("#editor/property_translations.gen.h", tlist)
     env.CommandNoCache(
         "#editor/property_translations.gen.h",
-        tlist,
-        env.Run(editor_builders.make_property_translations_header),
+        Glob("#editor/translations/properties/*"),
+        env.Run(editor_builders.make_translations_header),
     )
 
     # Documentation translations
-    tlist = glob.glob(env.Dir("#doc/translations").abspath + "/*.po")
-    env.Depends("#editor/doc_translations.gen.h", tlist)
     env.CommandNoCache(
         "#editor/doc_translations.gen.h",
-        tlist,
-        env.Run(editor_builders.make_doc_translations_header),
+        Glob("#doc/translations/*"),
+        env.Run(editor_builders.make_translations_header),
     )
 
     # Extractable translations
-    tlist = glob.glob(env.Dir("#editor/translations/extractable").abspath + "/*.po")
-    tlist.extend(glob.glob(env.Dir("#editor/translations/extractable").abspath + "/extractable.pot"))
-    env.Depends("#editor/extractable_translations.gen.h", tlist)
     env.CommandNoCache(
         "#editor/extractable_translations.gen.h",
-        tlist,
-        env.Run(editor_builders.make_extractable_translations_header),
+        Glob("#editor/translations/extractable/*"),
+        env.Run(editor_builders.make_translations_header),
     )
 
     env.add_source_files(env.editor_sources, "*.cpp")

editor/editor_builders.py

@@ -2,141 +2,95 @@
 
 import os
 import os.path
-import shutil
 import subprocess
 import tempfile
 import uuid
-import zlib
 
-from methods import print_warning
+import methods
 
 
 def make_doc_header(target, source, env):
-    dst = str(target[0])
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        buf = ""
-        docbegin = ""
-        docend = ""
-        for src in source:
-            src = str(src)
-            if not src.endswith(".xml"):
-                continue
-            with open(src, "r", encoding="utf-8") as f:
-                content = f.read()
-            buf += content
-
-        buf = (docbegin + buf + docend).encode("utf-8")
-        decomp_size = len(buf)
-
-        # Use maximum zlib compression level to further reduce file size
-        # (at the cost of initial build times).
-        buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
-
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef _DOC_DATA_RAW_H\n")
-        g.write("#define _DOC_DATA_RAW_H\n")
-        g.write('static const char *_doc_data_hash = "' + str(hash(buf)) + '";\n')
-        g.write("static const int _doc_data_compressed_size = " + str(len(buf)) + ";\n")
-        g.write("static const int _doc_data_uncompressed_size = " + str(decomp_size) + ";\n")
-        g.write("static const unsigned char _doc_data_compressed[] = {\n")
-        for i in range(len(buf)):
-            g.write("\t" + str(buf[i]) + ",\n")
-        g.write("};\n")
-
-        g.write("#endif")
-
-
-def make_translations_header(target, source, env, category):
-    dst = str(target[0])
-
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper()))
-        g.write("#define _{}_TRANSLATIONS_H\n".format(category.upper()))
-
-        sorted_paths = sorted([str(x) for x in source], key=lambda path: os.path.splitext(os.path.basename(path))[0])
-
-        msgfmt_available = shutil.which("msgfmt") is not None
-
-        if not msgfmt_available:
-            print_warning("msgfmt is not found, using .po files instead of .mo")
-
-        xl_names = []
-        for i in range(len(sorted_paths)):
-            name = os.path.splitext(os.path.basename(sorted_paths[i]))[0]
+    buffer = b"".join([methods.get_buffer(src) for src in map(str, source)])
+    decomp_size = len(buffer)
+    buffer = methods.compress_buffer(buffer)
+
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+inline constexpr const char *_doc_data_hash = "{hash(buffer)}";
+inline constexpr int _doc_data_compressed_size = {len(buffer)};
+inline constexpr int _doc_data_uncompressed_size = {decomp_size};
+inline constexpr const unsigned char _doc_data_compressed[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")
+
+
+def make_translations_header(target, source, env):
+    category = os.path.basename(str(target[0])).split("_")[0]
+    sorted_paths = sorted([src.abspath for src in source], key=lambda path: os.path.splitext(os.path.basename(path))[0])
+
+    xl_names = []
+    msgfmt = env.Detect("msgfmt")
+    if not msgfmt:
+        methods.print_warning("msgfmt not found, using .po files instead of .mo")
+
+    with methods.generated_wrapper(str(target[0])) as file:
+        for path in sorted_paths:
+            name = os.path.splitext(os.path.basename(path))[0]
             # msgfmt erases non-translated messages, so avoid using it if exporting the POT.
-            if msgfmt_available and name != category:
+            if msgfmt and name != category:
                 mo_path = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex + ".mo")
-                cmd = "msgfmt " + sorted_paths[i] + " --no-hash -o " + mo_path
+                cmd = f"{msgfmt} {path} --no-hash -o {mo_path}"
                 try:
                     subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE).communicate()
-                    with open(mo_path, "rb") as f:
-                        buf = f.read()
+                    buffer = methods.get_buffer(mo_path)
                 except OSError as e:
-                    print_warning(
+                    methods.print_warning(
                         "msgfmt execution failed, using .po file instead of .mo: path=%r; [%s] %s"
-                        % (sorted_paths[i], e.__class__.__name__, e)
+                        % (path, e.__class__.__name__, e)
                     )
-                    with open(sorted_paths[i], "rb") as f:
-                        buf = f.read()
+                    buffer = methods.get_buffer(path)
                 finally:
                     try:
-                        os.remove(mo_path)
+                        if os.path.exists(mo_path):
+                            os.remove(mo_path)
                     except OSError as e:
                         # Do not fail the entire build if it cannot delete a temporary file.
-                        print_warning(
+                        methods.print_warning(
                             "Could not delete temporary .mo file: path=%r; [%s] %s" % (mo_path, e.__class__.__name__, e)
                         )
             else:
-                with open(sorted_paths[i], "rb") as f:
-                    buf = f.read()
-
+                buffer = methods.get_buffer(path)
                 if name == category:
                     name = "source"
 
-            decomp_size = len(buf)
-            # Use maximum zlib compression level to further reduce file size
-            # (at the cost of initial build times).
-            buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
-
-            g.write("static const unsigned char _{}_translation_{}_compressed[] = {{\n".format(category, name))
-            for j in range(len(buf)):
-                g.write("\t" + str(buf[j]) + ",\n")
-
-            g.write("};\n")
+            decomp_size = len(buffer)
+            buffer = methods.compress_buffer(buffer)
 
-            xl_names.append([name, len(buf), str(decomp_size)])
-
-        g.write("struct {}TranslationList {{\n".format(category.capitalize()))
-        g.write("\tconst char* lang;\n")
-        g.write("\tint comp_size;\n")
-        g.write("\tint uncomp_size;\n")
-        g.write("\tconst unsigned char* data;\n")
-        g.write("};\n\n")
-        g.write("static {}TranslationList _{}_translations[] = {{\n".format(category.capitalize(), category))
-        for x in xl_names:
-            g.write(
-                '\t{{ "{}", {}, {}, _{}_translation_{}_compressed }},\n'.format(
-                    x[0], str(x[1]), str(x[2]), category, x[0]
-                )
-            )
-        g.write("\t{nullptr, 0, 0, nullptr}\n")
-        g.write("};\n")
+            file.write(f"""\
+inline constexpr const unsigned char _{category}_translation_{name}_compressed[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
 
-        g.write("#endif")
+""")
 
+            xl_names.append([name, len(buffer), decomp_size])
 
-def make_editor_translations_header(target, source, env):
-    make_translations_header(target, source, env, "editor")
+        file.write(f"""\
+struct {category.capitalize()}TranslationList {{
+	const char* lang;
+	int comp_size;
+	int uncomp_size;
+	const unsigned char* data;
+}};
 
+inline constexpr {category.capitalize()}TranslationList _{category}_translations[] = {{
+""")
 
-def make_property_translations_header(target, source, env):
-    make_translations_header(target, source, env, "property")
-
-
-def make_doc_translations_header(target, source, env):
-    make_translations_header(target, source, env, "doc")
-
+        for x in xl_names:
+            file.write(f'\t{{ "{x[0]}", {x[1]}, {x[2]}, _{category}_translation_{x[0]}_compressed }},\n')
 
-def make_extractable_translations_header(target, source, env):
-    make_translations_header(target, source, env, "extractable")
+        file.write("""\
+	{ nullptr, 0, 0, nullptr },
+};
+""")

editor/editor_translation.cpp

@@ -42,7 +42,7 @@
 Vector<String> get_editor_locales() {
 	Vector<String> locales;
 
-	EditorTranslationList *etl = _editor_translations;
+	const EditorTranslationList *etl = _editor_translations;
 	while (etl->data) {
 		const String &locale = etl->lang;
 		locales.push_back(locale);
@@ -56,7 +56,7 @@ Vector<String> get_editor_locales() {
 void load_editor_translations(const String &p_locale) {
 	const Ref<TranslationDomain> domain = TranslationServer::get_singleton()->get_or_add_domain("godot.editor");
 
-	EditorTranslationList *etl = _editor_translations;
+	const EditorTranslationList *etl = _editor_translations;
 	while (etl->data) {
 		if (etl->lang == p_locale) {
 			Vector<uint8_t> data;
@@ -84,7 +84,7 @@ void load_editor_translations(const String &p_locale) {
 void load_property_translations(const String &p_locale) {
 	const Ref<TranslationDomain> domain = TranslationServer::get_singleton()->get_or_add_domain("godot.properties");
 
-	PropertyTranslationList *etl = _property_translations;
+	const PropertyTranslationList *etl = _property_translations;
 	while (etl->data) {
 		if (etl->lang == p_locale) {
 			Vector<uint8_t> data;
@@ -112,7 +112,7 @@ void load_property_translations(const String &p_locale) {
 void load_doc_translations(const String &p_locale) {
 	const Ref<TranslationDomain> domain = TranslationServer::get_singleton()->get_or_add_domain("godot.documentation");
 
-	DocTranslationList *dtl = _doc_translations;
+	const DocTranslationList *dtl = _doc_translations;
 	while (dtl->data) {
 		if (dtl->lang == p_locale) {
 			Vector<uint8_t> data;
@@ -140,7 +140,7 @@ void load_doc_translations(const String &p_locale) {
 void load_extractable_translations(const String &p_locale) {
 	const Ref<TranslationDomain> domain = TranslationServer::get_singleton()->get_or_add_domain("godot.editor");
 
-	ExtractableTranslationList *etl = _extractable_translations;
+	const ExtractableTranslationList *etl = _extractable_translations;
 	while (etl->data) {
 		if (etl->lang == p_locale) {
 			Vector<uint8_t> data;
@@ -166,7 +166,7 @@ void load_extractable_translations(const String &p_locale) {
 }
 
 Vector<Vector<String>> get_extractable_message_list() {
-	ExtractableTranslationList *etl = _extractable_translations;
+	const ExtractableTranslationList *etl = _extractable_translations;
 	Vector<Vector<String>> list;
 
 	while (etl->data) {

editor/icons/editor_icons_builders.py

@@ -1,71 +1,47 @@
 """Functions used to generate source files during build time"""
 
 import os
-from io import StringIO
 
-from methods import to_raw_cstring
+import methods
 
 
 # See also `scene/theme/icons/default_theme_icons_builders.py`.
 def make_editor_icons_action(target, source, env):
-    dst = str(target[0])
-    svg_icons = source
-
-    with StringIO() as icons_string, StringIO() as s:
-        for svg in svg_icons:
-            with open(str(svg), "r") as svgf:
-                icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read()))
-
-        s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        s.write("#ifndef _EDITOR_ICONS_H\n")
-        s.write("#define _EDITOR_ICONS_H\n")
-        s.write("static const int editor_icons_count = {};\n".format(len(svg_icons)))
-        s.write("static const char *editor_icons_sources[] = {\n")
-        s.write(icons_string.getvalue())
-        s.write("};\n\n")
-        s.write("static const char *editor_icons_names[] = {\n")
-
-        # this is used to store the indices of thumbnail icons
-        thumb_medium_indices = []
-        thumb_big_indices = []
-        index = 0
-        for f in svg_icons:
-            fname = str(f)
-
-            # Trim the `.svg` extension from the string.
-            icon_name = os.path.basename(fname)[:-4]
-            # some special cases
-            if icon_name.endswith("MediumThumb"):  # don't know a better way to handle this
-                thumb_medium_indices.append(str(index))
-            if icon_name.endswith("BigThumb"):  # don't know a better way to handle this
-                thumb_big_indices.append(str(index))
-            if icon_name.endswith("GodotFile"):  # don't know a better way to handle this
-                thumb_big_indices.append(str(index))
-
-            s.write('\t"{0}"'.format(icon_name))
-
-            if fname != svg_icons[-1]:
-                s.write(",")
-            s.write("\n")
-
-            index += 1
-
-        s.write("};\n")
-
-        if thumb_medium_indices:
-            s.write("\n\n")
-            s.write("static const int editor_md_thumbs_count = {};\n".format(len(thumb_medium_indices)))
-            s.write("static const int editor_md_thumbs_indices[] = {")
-            s.write(", ".join(thumb_medium_indices))
-            s.write("};\n")
-        if thumb_big_indices:
-            s.write("\n\n")
-            s.write("static const int editor_bg_thumbs_count = {};\n".format(len(thumb_big_indices)))
-            s.write("static const int editor_bg_thumbs_indices[] = {")
-            s.write(", ".join(thumb_big_indices))
-            s.write("};\n")
-
-        s.write("#endif\n")
-
-        with open(dst, "w", encoding="utf-8", newline="\n") as f:
-            f.write(s.getvalue())
+    icons_names = []
+    icons_raw = []
+    icons_med = []
+    icons_big = []
+
+    for idx, svg in enumerate(source):
+        path = str(svg)
+        with open(path, encoding="utf-8", newline="\n") as file:
+            icons_raw.append(methods.to_raw_cstring(file.read()))
+
+        name = os.path.splitext(os.path.basename(path))[0]
+        icons_names.append(f'"{name}"')
+
+        if name.endswith("MediumThumb"):
+            icons_med.append(str(idx))
+        elif name.endswith(("BigThumb", "GodotFile")):
+            icons_big.append(str(idx))
+
+    icons_names_str = ",\n\t".join(icons_names)
+    icons_raw_str = ",\n\t".join(icons_raw)
+
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+inline constexpr int editor_icons_count = {len(icons_names)};
+inline constexpr const char *editor_icons_sources[] = {{
+	{icons_raw_str}
+}};
+
+inline constexpr const char *editor_icons_names[] = {{
+	{icons_names_str}
+}};
+
+inline constexpr int editor_md_thumbs_count = {len(icons_med)};
+inline constexpr int editor_md_thumbs_indices[] = {{ {", ".join(icons_med)} }};
+
+inline constexpr int editor_bg_thumbs_count = {len(icons_big)};
+inline constexpr int editor_bg_thumbs_indices[] = {{ {", ".join(icons_big)} }};
+""")

editor/template_builders.py

@@ -1,7 +1,8 @@
 """Functions used to generate source files during build time"""
 
 import os
-from io import StringIO
+
+import methods
 
 
 def parse_template(inherits, source, delimiter):
@@ -36,54 +37,36 @@ def parse_template(inherits, source, delimiter):
             script_template["script"].replace('"', '\\"').lstrip().replace("\n", "\\n").replace("\t", "_TS_")
         )
         return (
-            '{ String("'
-            + script_template["inherits"]
-            + '"), String("'
-            + script_template["name"]
-            + '"),  String("'
-            + script_template["description"]
-            + '"),  String("'
-            + script_template["script"]
-            + '")'
-            + " },\n"
+            f'{{ String("{script_template["inherits"]}"), '
+            + f'String("{script_template["name"]}"), '
+            + f'String("{script_template["description"]}"), '
+            + f'String("{script_template["script"]}") }},'
         )
 
 
 def make_templates(target, source, env):
-    dst = str(target[0])
-    with StringIO() as s:
-        s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n")
-        s.write("#ifndef _CODE_TEMPLATES_H\n")
-        s.write("#define _CODE_TEMPLATES_H\n\n")
-        s.write('#include "core/object/object.h"\n')
-        s.write('#include "core/object/script_language.h"\n')
-
-        delimiter = "#"  # GDScript single line comment delimiter by default.
-        if source:
-            ext = os.path.splitext(str(source[0]))[1]
-            if ext == ".cs":
-                delimiter = "//"
-
-        parsed_template_string = ""
-        number_of_templates = 0
+    delimiter = "#"  # GDScript single line comment delimiter by default.
+    if source:
+        ext = os.path.splitext(str(source[0]))[1]
+        if ext == ".cs":
+            delimiter = "//"
 
-        for filepath in source:
-            filepath = str(filepath)
-            node_name = os.path.basename(os.path.dirname(filepath))
-            parsed_template = parse_template(node_name, filepath, delimiter)
-            parsed_template_string += "\t" + parsed_template
-            number_of_templates += 1
-
-        s.write("\nstatic const int TEMPLATES_ARRAY_SIZE = " + str(number_of_templates) + ";\n")
-        s.write(
-            "\nstatic const struct ScriptLanguage::ScriptTemplate TEMPLATES[" + str(number_of_templates) + "] = {\n"
-        )
+    parsed_templates = []
 
-        s.write(parsed_template_string)
+    for filepath in source:
+        filepath = str(filepath)
+        node_name = os.path.basename(os.path.dirname(filepath))
+        parsed_templates.append(parse_template(node_name, filepath, delimiter))
 
-        s.write("};\n")
+    parsed_template_string = "\n\t".join(parsed_templates)
 
-        s.write("\n#endif\n")
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+#include "core/object/object.h"
+#include "core/object/script_language.h"
 
-        with open(dst, "w", encoding="utf-8", newline="\n") as f:
-            f.write(s.getvalue())
+inline constexpr int TEMPLATES_ARRAY_SIZE = {len(parsed_templates)};
+static const struct ScriptLanguage::ScriptTemplate TEMPLATES[TEMPLATES_ARRAY_SIZE] = {{
+	{parsed_template_string}
+}};
+""")

editor/themes/SCsub

@@ -3,17 +3,14 @@ from misc.utility.scons_hints import *
 
 Import("env")
 
-import glob
-
 import editor_theme_builders
 
 # Fonts
-flist = glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.ttf")
-flist.extend(glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.otf"))
-flist.extend(glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.woff"))
-flist.extend(glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.woff2"))
+flist = Glob("#thirdparty/fonts/*.ttf")
+flist.extend(Glob("#thirdparty/fonts/*.otf"))
+flist.extend(Glob("#thirdparty/fonts/*.woff"))
+flist.extend(Glob("#thirdparty/fonts/*.woff2"))
 flist.sort()
-env.Depends("#editor/themes/builtin_fonts.gen.h", flist)
 env.CommandNoCache(
     "#editor/themes/builtin_fonts.gen.h",
     flist,

editor/themes/editor_theme_builders.py

@@ -2,28 +2,20 @@
 
 import os
 
+import methods
 
-def make_fonts_header(target, source, env):
-    dst = str(target[0])
-
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef _EDITOR_FONTS_H\n")
-        g.write("#define _EDITOR_FONTS_H\n")
-
-        # Saving uncompressed, since FreeType will reference from memory pointer.
-        for i in range(len(source)):
-            file = str(source[i])
-            with open(file, "rb") as f:
-                buf = f.read()
-
-            name = os.path.splitext(os.path.basename(file))[0]
 
-            g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n")
-            g.write("static const unsigned char _font_" + name + "[] = {\n")
-            for j in range(len(buf)):
-                g.write("\t" + str(buf[j]) + ",\n")
-
-            g.write("};\n")
-
-        g.write("#endif")
+def make_fonts_header(target, source, env):
+    with methods.generated_wrapper(str(target[0])) as file:
+        for src in map(str, source):
+            # Saving uncompressed, since FreeType will reference from memory pointer.
+            buffer = methods.get_buffer(src)
+            name = os.path.splitext(os.path.basename(src))[0]
+
+            file.write(f"""\
+inline constexpr int _font_{name}_size = {len(buffer)};
+inline constexpr unsigned char _font_{name}[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+
+""")

main/SCsub

@@ -17,7 +17,6 @@ if env["steamapi"] and env.editor_build:
 if env["tests"]:
     env_main.Append(CPPDEFINES=["TESTS_ENABLED"])
 
-env_main.Depends("#main/splash.gen.h", "#main/splash.png")
 env_main.CommandNoCache(
     "#main/splash.gen.h",
     "#main/splash.png",
@@ -25,14 +24,12 @@ env_main.CommandNoCache(
 )
 
 if env_main.editor_build and not env_main["no_editor_splash"]:
-    env_main.Depends("#main/splash_editor.gen.h", "#main/splash_editor.png")
     env_main.CommandNoCache(
         "#main/splash_editor.gen.h",
         "#main/splash_editor.png",
         env.Run(main_builders.make_splash_editor),
     )
 
-env_main.Depends("#main/app_icon.gen.h", "#main/app_icon.png")
 env_main.CommandNoCache(
     "#main/app_icon.gen.h",
     "#main/app_icon.png",

main/main_builders.py

@@ -1,60 +1,42 @@
 """Functions used to generate source files during build time"""
 
+import methods
 
-def make_splash(target, source, env):
-    src = str(source[0])
-    dst = str(target[0])
 
-    with open(src, "rb") as f:
-        buf = f.read()
+def make_splash(target, source, env):
+    buffer = methods.get_buffer(str(source[0]))
 
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef BOOT_SPLASH_H\n")
-        g.write("#define BOOT_SPLASH_H\n")
+    with methods.generated_wrapper(str(target[0])) as file:
         # Use a neutral gray color to better fit various kinds of projects.
-        g.write("static const Color boot_splash_bg_color = Color(0.14, 0.14, 0.14);\n")
-        g.write("static const unsigned char boot_splash_png[] = {\n")
-        for i in range(len(buf)):
-            g.write(str(buf[i]) + ",\n")
-        g.write("};\n")
-        g.write("#endif")
+        file.write(f"""\
+static const Color boot_splash_bg_color = Color(0.14, 0.14, 0.14);
+inline constexpr const unsigned char boot_splash_png[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")
 
 
 def make_splash_editor(target, source, env):
-    src = str(source[0])
-    dst = str(target[0])
-
-    with open(src, "rb") as f:
-        buf = f.read()
+    buffer = methods.get_buffer(str(source[0]))
 
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef BOOT_SPLASH_EDITOR_H\n")
-        g.write("#define BOOT_SPLASH_EDITOR_H\n")
+    with methods.generated_wrapper(str(target[0])) as file:
         # The editor splash background color is taken from the default editor theme's background color.
         # This helps achieve a visually "smoother" transition between the splash screen and the editor.
-        g.write("static const Color boot_splash_editor_bg_color = Color(0.125, 0.145, 0.192);\n")
-        g.write("static const unsigned char boot_splash_editor_png[] = {\n")
-        for i in range(len(buf)):
-            g.write(str(buf[i]) + ",\n")
-        g.write("};\n")
-        g.write("#endif")
+        file.write(f"""\
+static const Color boot_splash_editor_bg_color = Color(0.125, 0.145, 0.192);
+inline constexpr const unsigned char boot_splash_editor_png[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")
 
 
 def make_app_icon(target, source, env):
-    src = str(source[0])
-    dst = str(target[0])
-
-    with open(src, "rb") as f:
-        buf = f.read()
-
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef APP_ICON_H\n")
-        g.write("#define APP_ICON_H\n")
-        g.write("static const unsigned char app_icon_png[] = {\n")
-        for i in range(len(buf)):
-            g.write(str(buf[i]) + ",\n")
-        g.write("};\n")
-        g.write("#endif")
+    buffer = methods.get_buffer(str(source[0]))
+
+    with methods.generated_wrapper(str(target[0])) as file:
+        # Use a neutral gray color to better fit various kinds of projects.
+        file.write(f"""\
+inline constexpr const unsigned char app_icon_png[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")

methods.py

@@ -6,6 +6,8 @@ import os
 import re
 import subprocess
 import sys
+import textwrap
+import zlib
 from collections import OrderedDict
 from io import StringIO, TextIOBase
 from pathlib import Path
@@ -144,30 +146,36 @@ def get_version_info(module_version_string="", silent=False):
         if not silent:
             print_info(f"Using version status '{version_info['status']}', overriding the original '{version.status}'.")
 
+    return version_info
+
+
+def get_git_info():
+    os.chdir(base_folder_path)
+
     # Parse Git hash if we're in a Git repo.
-    githash = ""
-    gitfolder = ".git"
+    git_hash = ""
+    git_folder = ".git"
 
     if os.path.isfile(".git"):
         with open(".git", "r", encoding="utf-8") as file:
             module_folder = file.readline().strip()
         if module_folder.startswith("gitdir: "):
-            gitfolder = module_folder[8:]
+            git_folder = module_folder[8:]
 
-    if os.path.isfile(os.path.join(gitfolder, "HEAD")):
-        with open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8") as file:
+    if os.path.isfile(os.path.join(git_folder, "HEAD")):
+        with open(os.path.join(git_folder, "HEAD"), "r", encoding="utf8") as file:
             head = file.readline().strip()
         if head.startswith("ref: "):
             ref = head[5:]
             # If this directory is a Git worktree instead of a root clone.
-            parts = gitfolder.split("/")
+            parts = git_folder.split("/")
             if len(parts) > 2 and parts[-2] == "worktrees":
-                gitfolder = "/".join(parts[0:-2])
-            head = os.path.join(gitfolder, ref)
-            packedrefs = os.path.join(gitfolder, "packed-refs")
+                git_folder = "/".join(parts[0:-2])
+            head = os.path.join(git_folder, ref)
+            packedrefs = os.path.join(git_folder, "packed-refs")
             if os.path.isfile(head):
                 with open(head, "r", encoding="utf-8") as file:
-                    githash = file.readline().strip()
+                    git_hash = file.readline().strip()
             elif os.path.isfile(packedrefs):
                 # Git may pack refs into a single file. This code searches .git/packed-refs file for the current ref's hash.
                 # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-pack-refs.html
@@ -176,26 +184,26 @@ def get_version_info(module_version_string="", silent=False):
                         continue
                     (line_hash, line_ref) = line.split(" ")
                     if ref == line_ref:
-                        githash = line_hash
+                        git_hash = line_hash
                         break
         else:
-            githash = head
-
-    version_info["git_hash"] = githash
-    # Fallback to 0 as a timestamp (will be treated as "unknown" in the engine).
-    version_info["git_timestamp"] = 0
+            git_hash = head
 
     # Get the UNIX timestamp of the build commit.
+    git_timestamp = 0
     if os.path.exists(".git"):
         try:
-            version_info["git_timestamp"] = subprocess.check_output(
-                ["git", "log", "-1", "--pretty=format:%ct", "--no-show-signature", githash]
-            ).decode("utf-8")
+            git_timestamp = subprocess.check_output(
+                ["git", "log", "-1", "--pretty=format:%ct", "--no-show-signature", git_hash], encoding="utf-8"
+            )
         except (subprocess.CalledProcessError, OSError):
             # `git` not found in PATH.
             pass
 
-    return version_info
+    return {
+        "git_hash": git_hash,
+        "git_timestamp": git_timestamp,
+    }
 
 
 def get_cmdline_bool(option, default):
@@ -1417,6 +1425,11 @@ def generate_vs_project(env, original_args, project_name="godot"):
         sys.exit()
 
 
+############################################################
+# FILE GENERATION & FORMATTING
+############################################################
+
+
 def generate_copyright_header(filename: str) -> str:
     MARGIN = 70
     TEMPLATE = """\
@@ -1450,15 +1463,14 @@ def generate_copyright_header(filename: str) -> str:
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
 /**************************************************************************/
 """
-    filename = filename.split("/")[-1].ljust(MARGIN)
-    if len(filename) > MARGIN:
+    if len(filename := os.path.basename(filename).ljust(MARGIN)) > MARGIN:
         print_warning(f'Filename "{filename}" too large for copyright header.')
     return TEMPLATE % filename
 
 
 @contextlib.contextmanager
 def generated_wrapper(
-    path,  # FIXME: type with `Union[str, Node, List[Node]]` when pytest conflicts are resolved
+    path: str,
     guard: Optional[bool] = None,
 ) -> Generator[TextIOBase, None, None]:
     """
@@ -1466,26 +1478,11 @@ def generated_wrapper(
     for generated scripts. Meant to be invoked via `with` statement similar to
     creating a file.
 
-    - `path`: The path of the file to be created. Can be passed a raw string, an
-    isolated SCons target, or a full SCons target list. If a target list contains
-    multiple entries, produces a warning & only creates the first entry.
+    - `path`: The path of the file to be created.
     - `guard`: Optional bool to determine if `#pragma once` should be added. If
     unassigned, the value is determined by file extension.
     """
 
-    # Handle unfiltered SCons target[s] passed as path.
-    if not isinstance(path, str):
-        if isinstance(path, list):
-            if len(path) > 1:
-                print_warning(
-                    f"Attempting to use generated wrapper with multiple targets; will only use first entry: {path[0]}"
-                )
-            path = path[0]
-        if not hasattr(path, "get_abspath"):
-            raise TypeError(f'Expected type "str", "Node" or "List[Node]"; was passed {type(path)}.')
-        path = path.get_abspath()
-
-    path = str(path).replace("\\", "/")
     if guard is None:
         guard = path.endswith((".h", ".hh", ".hpp", ".hxx", ".inc"))
 
@@ -1503,6 +1500,50 @@ def generated_wrapper(
         file.write("\n")
 
 
+def get_buffer(path: str) -> bytes:
+    with open(path, "rb") as file:
+        return file.read()
+
+
+def compress_buffer(buffer: bytes) -> bytes:
+    # Use maximum zlib compression level to further reduce file size
+    # (at the cost of initial build times).
+    return zlib.compress(buffer, zlib.Z_BEST_COMPRESSION)
+
+
+def format_buffer(buffer: bytes, indent: int = 0, width: int = 120, initial_indent: bool = False) -> str:
+    return textwrap.fill(
+        ", ".join(str(byte) for byte in buffer),
+        width=width,
+        initial_indent="\t" * indent if initial_indent else "",
+        subsequent_indent="\t" * indent,
+        tabsize=4,
+    )
+
+
+############################################################
+# CSTRING PARSING
+############################################################
+
+C_ESCAPABLES = [
+    ("\\", "\\\\"),
+    ("\a", "\\a"),
+    ("\b", "\\b"),
+    ("\f", "\\f"),
+    ("\n", "\\n"),
+    ("\r", "\\r"),
+    ("\t", "\\t"),
+    ("\v", "\\v"),
+    # ("'", "\\'"),  # Skip, as we're only dealing with full strings.
+    ('"', '\\"'),
+]
+C_ESCAPE_TABLE = str.maketrans(dict((x, y) for x, y in C_ESCAPABLES))
+
+
+def to_escaped_cstring(value: str) -> str:
+    return value.translate(C_ESCAPE_TABLE)
+
+
 def to_raw_cstring(value: Union[str, List[str]]) -> str:
     MAX_LITERAL = 16 * 1024
 
@@ -1540,4 +1581,8 @@ def to_raw_cstring(value: Union[str, List[str]]) -> str:
 
         split += [segment]
 
-    return " ".join(f'R"<!>({x.decode()})<!>"' for x in split)
+    if len(split) == 1:
+        return f'R"<!>({split[0].decode()})<!>"'
+    else:
+        # Wrap multiple segments in parenthesis to suppress `string-concatenation` warnings on clang.
+        return "({})".format(" ".join(f'R"<!>({segment.decode()})<!>"' for segment in split))

modules/SCsub

@@ -17,8 +17,9 @@ Export("env_modules")
 
 # Header with MODULE_*_ENABLED defines.
 def modules_enabled_builder(target, source, env):
-    with methods.generated_wrapper(target) as file:
-        for module in source[0].read():
+    modules = sorted(source[0].read())
+    with methods.generated_wrapper(str(target[0])) as file:
+        for module in modules:
             file.write(f"#define MODULE_{module.upper()}_ENABLED\n")
 
 
@@ -29,14 +30,26 @@ modules_enabled = env.CommandNoCache(
 
 def register_module_types_builder(target, source, env):
     modules = source[0].read()
-    mod_inc = "\n".join([f'#include "{p}/register_types.h"' for p in modules.values()])
+    mod_inc = "\n".join([f'#include "{value}/register_types.h"' for value in modules.values()])
     mod_init = "\n".join(
-        [f"#ifdef MODULE_{n.upper()}_ENABLED\n\tinitialize_{n}_module(p_level);\n#endif" for n in modules.keys()]
+        [
+            f"""\
+#ifdef MODULE_{key.upper()}_ENABLED
+	initialize_{key}_module(p_level);
+#endif"""
+            for key in modules.keys()
+        ]
     )
     mod_uninit = "\n".join(
-        [f"#ifdef MODULE_{n.upper()}_ENABLED\n\tuninitialize_{n}_module(p_level);\n#endif" for n in modules.keys()]
+        [
+            f"""\
+#ifdef MODULE_{key.upper()}_ENABLED
+	uninitialize_{key}_module(p_level);
+#endif"""
+            for key in modules.keys()
+        ]
     )
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             f"""\
 #include "register_module_types.h"
@@ -88,9 +101,10 @@ for name, path in env.module_list.items():
 if env["tests"]:
 
     def modules_tests_builder(target, source, env):
-        with methods.generated_wrapper(target) as file:
-            for header in source:
-                file.write('#include "{}"\n'.format(os.path.normpath(header.path).replace("\\", "/")))
+        headers = sorted([os.path.relpath(src.path, methods.base_folder_path).replace("\\", "/") for src in source])
+        with methods.generated_wrapper(str(target[0])) as file:
+            for header in headers:
+                file.write(f'#include "{header}"\n')
 
     env.CommandNoCache("modules_tests.gen.h", test_headers, env.Run(modules_tests_builder))
 

modules/text_server_adv/SCsub

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 from misc.utility.scons_hints import *
 
+import methods
+
 Import("env")
 Import("env_modules")
 
@@ -8,28 +10,21 @@ env_text_server_adv = env_modules.Clone()
 
 
 def make_icu_data(target, source, env):
-    dst = target[0].srcnode().abspath
-
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("/* (C) 2016 and later: Unicode, Inc. and others. */\n")
-        g.write("/* License & terms of use: https://www.unicode.org/copyright.html */\n")
-        g.write("#ifndef _ICU_DATA_H\n")
-        g.write("#define _ICU_DATA_H\n")
-        g.write('#include "unicode/utypes.h"\n')
-        g.write('#include "unicode/udata.h"\n')
-        g.write('#include "unicode/uversion.h"\n')
-
-        with open(source[0].srcnode().abspath, "rb") as f:
-            buf = f.read()
-
-        g.write('extern "C" U_EXPORT const size_t U_ICUDATA_SIZE = ' + str(len(buf)) + ";\n")
-        g.write('extern "C" U_EXPORT const unsigned char U_ICUDATA_ENTRY_POINT[] = {\n')
-        for i in range(len(buf)):
-            g.write("\t" + str(buf[i]) + ",\n")
-
-        g.write("};\n")
-        g.write("#endif")
+    buffer = methods.get_buffer(str(source[0]))
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+/* (C) 2016 and later: Unicode, Inc. and others. */
+/* License & terms of use: https://www.unicode.org/copyright.html */
+
+#include <unicode/utypes.h>
+#include <unicode/udata.h>
+#include <unicode/uversion.h>
+
+extern "C" U_EXPORT const size_t U_ICUDATA_SIZE = {len(buffer)};
+extern "C" U_EXPORT const unsigned char U_ICUDATA_ENTRY_POINT[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")
 
 
 # Thirdparty source files

platform/SCsub

@@ -18,10 +18,10 @@ def export_icon_builder(target, source, env):
     platform = src_path.parent.parent.stem
     with open(str(source[0]), "r") as file:
         svg = file.read()
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             f"""\
-static const char *_{platform}_{src_name}_svg = {methods.to_raw_cstring(svg)};
+inline constexpr const char *_{platform}_{src_name}_svg = {methods.to_raw_cstring(svg)};
 """
         )
 
@@ -37,7 +37,7 @@ def register_platform_apis_builder(target, source, env):
     api_inc = "\n".join([f'#include "{p}/api/api.h"' for p in platforms])
     api_reg = "\n".join([f"\tregister_{p}_api();" for p in platforms])
     api_unreg = "\n".join([f"\tunregister_{p}_api();" for p in platforms])
-    with methods.generated_wrapper(target) as file:
+    with methods.generated_wrapper(str(target[0])) as file:
         file.write(
             f"""\
 #include "register_platform_apis.h"

scene/theme/SCsub

@@ -9,7 +9,6 @@ env.add_source_files(env.scene_sources, "*.cpp")
 
 SConscript("icons/SCsub")
 
-env.Depends("#scene/theme/default_font.gen.h", "#thirdparty/fonts/OpenSans_SemiBold.woff2")
 env.CommandNoCache(
     "#scene/theme/default_font.gen.h",
     "#thirdparty/fonts/OpenSans_SemiBold.woff2",

scene/theme/default_theme_builders.py

@@ -1,30 +1,21 @@
 """Functions used to generate source files during build time"""
 
 import os
-import os.path
 
+import methods
 
-def make_fonts_header(target, source, env):
-    dst = str(target[0])
-
-    with open(dst, "w", encoding="utf-8", newline="\n") as g:
-        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-        g.write("#ifndef _DEFAULT_FONTS_H\n")
-        g.write("#define _DEFAULT_FONTS_H\n")
-
-        # Saving uncompressed, since FreeType will reference from memory pointer.
-        for i in range(len(source)):
-            file = str(source[i])
-            with open(file, "rb") as f:
-                buf = f.read()
-
-            name = os.path.splitext(os.path.basename(file))[0]
 
-            g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n")
-            g.write("static const unsigned char _font_" + name + "[] = {\n")
-            for j in range(len(buf)):
-                g.write("\t" + str(buf[j]) + ",\n")
-
-            g.write("};\n")
-
-        g.write("#endif")
+def make_fonts_header(target, source, env):
+    with methods.generated_wrapper(str(target[0])) as file:
+        for src in map(str, source):
+            # Saving uncompressed, since FreeType will reference from memory pointer.
+            buffer = methods.get_buffer(src)
+            name = os.path.splitext(os.path.basename(src))[0]
+
+            file.write(f"""\
+inline constexpr int _font_{name}_size = {len(buffer)};
+inline constexpr unsigned char _font_{name}[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+
+""")

scene/theme/icons/default_theme_icons_builders.py

@@ -1,51 +1,35 @@
 """Functions used to generate source files during build time"""
 
 import os
-from io import StringIO
 
-from methods import to_raw_cstring
+import methods
 
 
 # See also `editor/icons/editor_icons_builders.py`.
 def make_default_theme_icons_action(target, source, env):
-    dst = str(target[0])
-    svg_icons = [str(x) for x in source]
+    icons_names = []
+    icons_raw = []
 
-    with StringIO() as icons_string, StringIO() as s:
-        for svg in svg_icons:
-            with open(svg, "r") as svgf:
-                icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read()))
+    for src in map(str, source):
+        with open(src, encoding="utf-8", newline="\n") as file:
+            icons_raw.append(methods.to_raw_cstring(file.read()))
 
-        s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n")
-        s.write('#include "modules/modules_enabled.gen.h"\n\n')
-        s.write("#ifndef _DEFAULT_THEME_ICONS_H\n")
-        s.write("#define _DEFAULT_THEME_ICONS_H\n")
-        s.write("static const int default_theme_icons_count = {};\n\n".format(len(svg_icons)))
-        s.write("#ifdef MODULE_SVG_ENABLED\n")
-        s.write("static const char *default_theme_icons_sources[] = {\n")
-        s.write(icons_string.getvalue())
-        s.write("};\n")
-        s.write("#endif // MODULE_SVG_ENABLED\n\n")
-        s.write("static const char *default_theme_icons_names[] = {\n")
+        name = os.path.splitext(os.path.basename(src))[0]
+        icons_names.append(f'"{name}"')
 
-        index = 0
-        for f in svg_icons:
-            fname = str(f)
+    icons_names_str = ",\n\t".join(icons_names)
+    icons_raw_str = ",\n\t".join(icons_raw)
 
-            # Trim the `.svg` extension from the string.
-            icon_name = os.path.basename(fname)[:-4]
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+#include "modules/modules_enabled.gen.h"
 
-            s.write('\t"{0}"'.format(icon_name))
+inline constexpr int default_theme_icons_count = {len(icons_names)};
+inline constexpr const char *default_theme_icons_sources[] = {{
+	{icons_raw_str}
+}};
 
-            if fname != svg_icons[-1]:
-                s.write(",")
-            s.write("\n")
-
-            index += 1
-
-        s.write("};\n")
-
-        s.write("#endif\n")
-
-        with open(dst, "w", encoding="utf-8", newline="\n") as f:
-            f.write(s.getvalue())
+inline constexpr const char *default_theme_icons_names[] = {{
+	{icons_names_str}
+}};
+""")