SCons: Add method to generate raw cstrings

editor/icons/editor_icons_builders.py

@@ -3,6 +3,8 @@
 import os
 from io import StringIO
 
+from methods import to_raw_cstring
+
 
 # See also `scene/theme/icons/default_theme_icons_builders.py`.
 def make_editor_icons_action(target, source, env):
@@ -10,21 +12,9 @@ def make_editor_icons_action(target, source, env):
     svg_icons = source
 
     with StringIO() as icons_string, StringIO() as s:
-        for f in svg_icons:
-            fname = str(f)
-
-            icons_string.write('\t"')
-
-            with open(fname, "rb") as svgf:
-                b = svgf.read(1)
-                while len(b) == 1:
-                    icons_string.write("\\" + str(hex(ord(b)))[1:])
-                    b = svgf.read(1)
-
-            icons_string.write('"')
-            if fname != svg_icons[-1]:
-                icons_string.write(",")
-            icons_string.write("\n")
+        for svg in svg_icons:
+            with open(str(svg), "r") as svgf:
+                icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read()))
 
         s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
         s.write("#ifndef _EDITOR_ICONS_H\n")

gles3_builders.py

@@ -3,7 +3,7 @@
 import os.path
 from typing import Optional
 
-from methods import print_error
+from methods import print_error, to_raw_cstring
 
 
 class GLES3HeaderStruct:
@@ -553,20 +553,12 @@ def build_gles3_header(
             fd.write("\t\tstatic const Feedback* _feedbacks=nullptr;\n")
 
         fd.write("\t\tstatic const char _vertex_code[]={\n")
-        for x in header_data.vertex_lines:
-            for c in x:
-                fd.write(str(ord(c)) + ",")
-
-            fd.write(str(ord("\n")) + ",")
-        fd.write("\t\t0};\n\n")
+        fd.write(to_raw_cstring(header_data.vertex_lines))
+        fd.write("\n\t\t};\n\n")
 
         fd.write("\t\tstatic const char _fragment_code[]={\n")
-        for x in header_data.fragment_lines:
-            for c in x:
-                fd.write(str(ord(c)) + ",")
-
-            fd.write(str(ord("\n")) + ",")
-        fd.write("\t\t0};\n\n")
+        fd.write(to_raw_cstring(header_data.fragment_lines))
+        fd.write("\n\t\t};\n\n")
 
         fd.write(
             '\t\t_setup(_vertex_code,_fragment_code,"'

glsl_builders.py

@@ -1,25 +1,9 @@
 """Functions used to generate source files during build time"""
 
 import os.path
-from typing import Iterable, Optional
+from typing import Optional
 
-from methods import print_error
-
-
-def generate_inline_code(input_lines: Iterable[str], insert_newline: bool = True):
-    """Take header data and generate inline code
-
-    :param: input_lines: values for shared inline code
-    :return: str - generated inline value
-    """
-    output = []
-    for line in input_lines:
-        if line:
-            output.append(",".join(str(ord(c)) for c in line))
-        if insert_newline:
-            output.append("%s" % ord("\n"))
-    output.append("0")
-    return ",".join(output)
+from methods import print_error, to_raw_cstring
 
 
 class RDHeaderStruct:
@@ -127,13 +111,13 @@ def build_rd_header(
 
     if header_data.compute_lines:
         body_parts = [
-            "static const char _compute_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.compute_lines),
+            "static const char _compute_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.compute_lines),
             f'setup(nullptr, nullptr, _compute_code, "{out_file_class}");',
         ]
     else:
         body_parts = [
-            "static const char _vertex_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.vertex_lines),
-            "static const char _fragment_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.fragment_lines),
+            "static const char _vertex_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.vertex_lines),
+            "static const char _fragment_code[] = {\n%s\n\t\t};" % to_raw_cstring(header_data.fragment_lines),
             f'setup(_vertex_code, _fragment_code, nullptr, "{out_file_class}");',
         ]
 
@@ -211,7 +195,7 @@ def build_raw_header(
 #define {out_file_ifdef}_RAW_H
 
 static const char {out_file_base}[] = {{
-    {generate_inline_code(header_data.code, insert_newline=False)}
+{to_raw_cstring(header_data.code)}
 }};
 #endif
 """

methods.py

@@ -8,7 +8,7 @@ from collections import OrderedDict
 from enum import Enum
 from io import StringIO, TextIOWrapper
 from pathlib import Path
-from typing import Generator, Optional
+from typing import Generator, List, Optional, Union
 
 # Get the "Godot" folder name ahead of time
 base_folder_path = str(os.path.abspath(Path(__file__).parent)) + "/"
@@ -1641,3 +1641,43 @@ def generated_wrapper(
             file.write(f"\n\n#endif // {header_guard}")
 
         file.write("\n")
+
+
+def to_raw_cstring(value: Union[str, List[str]]) -> str:
+    MAX_LITERAL = 16 * 1024
+
+    if isinstance(value, list):
+        value = "\n".join(value) + "\n"
+
+    split: List[bytes] = []
+    offset = 0
+    encoded = value.encode()
+
+    while offset <= len(encoded):
+        segment = encoded[offset : offset + MAX_LITERAL]
+        offset += MAX_LITERAL
+        if len(segment) == MAX_LITERAL:
+            # Try to segment raw strings at double newlines to keep readable.
+            pretty_break = segment.rfind(b"\n\n")
+            if pretty_break != -1:
+                segment = segment[: pretty_break + 1]
+                offset -= MAX_LITERAL - pretty_break - 1
+            # If none found, ensure we end with valid utf8.
+            # https://github.com/halloleo/unicut/blob/master/truncate.py
+            elif segment[-1] & 0b10000000:
+                last_11xxxxxx_index = [i for i in range(-1, -5, -1) if segment[i] & 0b11000000 == 0b11000000][0]
+                last_11xxxxxx = segment[last_11xxxxxx_index]
+                if not last_11xxxxxx & 0b00100000:
+                    last_char_length = 2
+                elif not last_11xxxxxx & 0b0010000:
+                    last_char_length = 3
+                elif not last_11xxxxxx & 0b0001000:
+                    last_char_length = 4
+
+                if last_char_length > -last_11xxxxxx_index:
+                    segment = segment[:last_11xxxxxx_index]
+                    offset += last_11xxxxxx_index
+
+        split += [segment]
+
+    return " ".join(f'R"<!>({x.decode()})<!>"' for x in split)

platform/SCsub

@@ -15,12 +15,12 @@ def export_icon_builder(target, source, env):
     src_path = Path(str(source[0]))
     src_name = src_path.stem
     platform = src_path.parent.parent.stem
-    with open(str(source[0]), "rb") as file:
-        svg = "".join([f"\\{hex(x)[1:]}" for x in file.read()])
+    with open(str(source[0]), "r") as file:
+        svg = file.read()
     with methods.generated_wrapper(target, prefix=platform) as file:
         file.write(
             f"""\
-static const char *_{platform}_{src_name}_svg = "{svg}";
+static const char *_{platform}_{src_name}_svg = {methods.to_raw_cstring(svg)};
 """
         )
 

scene/theme/icons/default_theme_icons_builders.py

@@ -3,6 +3,8 @@
 import os
 from io import StringIO
 
+from methods import to_raw_cstring
+
 
 # See also `editor/icons/editor_icons_builders.py`.
 def make_default_theme_icons_action(target, source, env):
@@ -10,21 +12,9 @@ def make_default_theme_icons_action(target, source, env):
     svg_icons = [str(x) for x in source]
 
     with StringIO() as icons_string, StringIO() as s:
-        for f in svg_icons:
-            fname = str(f)
-
-            icons_string.write('\t"')
-
-            with open(fname, "rb") as svgf:
-                b = svgf.read(1)
-                while len(b) == 1:
-                    icons_string.write("\\" + str(hex(ord(b)))[1:])
-                    b = svgf.read(1)
-
-            icons_string.write('"')
-            if fname != svg_icons[-1]:
-                icons_string.write(",")
-            icons_string.write("\n")
+        for svg in svg_icons:
+            with open(svg, "r") as svgf:
+                icons_string.write("\t%s,\n" % to_raw_cstring(svgf.read()))
 
         s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n")
         s.write('#include "modules/modules_enabled.gen.h"\n\n')

tests/python_build/fixtures/gles3/vertex_fragment_expected_full.glsl

@@ -37,10 +37,34 @@ protected:
 
 		static const Feedback* _feedbacks=nullptr;
 		static const char _vertex_code[]={
-10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,105,110,32,104,105,103,104,112,32,118,101,99,51,32,118,101,114,116,101,120,59,10,10,111,117,116,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,32,61,32,118,101,99,52,40,118,101,114,116,101,120,46,120,44,49,44,48,44,49,41,59,10,125,10,10,		0};
+R"<!>(
+precision highp float;
+precision highp int;
+
+layout(location = 0) in highp vec3 vertex;
+
+out highp vec4 position_interp;
+
+void main() {
+	position_interp = vec4(vertex.x,1,0,1);
+}
+
+)<!>"
+		};
 
 		static const char _fragment_code[]={
-10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,105,110,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,104,105,103,104,112,32,102,108,111,97,116,32,100,101,112,116,104,32,61,32,40,40,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,122,32,47,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,119,41,32,43,32,49,46,48,41,59,10,9,102,114,97,103,95,99,111,108,111,114,32,61,32,118,101,99,52,40,100,101,112,116,104,41,59,10,125,10,		0};
+R"<!>(
+precision highp float;
+precision highp int;
+
+in highp vec4 position_interp;
+
+void main() {
+	highp float depth = ((position_interp.z / position_interp.w) + 1.0);
+	frag_color = vec4(depth);
+}
+)<!>"
+		};
 
 		_setup(_vertex_code,_fragment_code,"VertexFragmentShaderGLES3",0,_uniform_strings,0,_ubo_pairs,0,_feedbacks,0,_texunit_pairs,1,_spec_pairs,1,_variant_defines);
 	}

tests/python_build/fixtures/glsl/compute_expected_full.glsl

@@ -3,6 +3,18 @@
 #define COMPUTE_SHADER_GLSL_RAW_H
 
 static const char compute_shader_glsl[] = {
-    35,91,99,111,109,112,117,116,101,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,118,101,99,51,32,115,116,97,116,105,99,95,108,105,103,104,116,32,61,32,118,101,99,51,40,48,44,32,49,44,32,48,41,59,10,125,10,0
+R"<!>(#[compute]
+
+#version 450
+
+#VERSION_DEFINES
+
+
+#define M_PI 3.14159265359
+
+void main() {
+	vec3 static_light = vec3(0, 1, 0);
+}
+)<!>"
 };
 #endif

tests/python_build/fixtures/glsl/vertex_fragment_expected_full.glsl

@@ -3,6 +3,38 @@
 #define VERTEX_FRAGMENT_SHADER_GLSL_RAW_H
 
 static const char vertex_fragment_shader_glsl[] = {
-    35,91,118,101,114,115,105,111,110,115,93,10,10,108,105,110,101,115,32,61,32,34,35,100,101,102,105,110,101,32,77,79,68,69,95,76,73,78,69,83,34,59,10,10,35,91,118,101,114,116,101,120,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,51,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,10,35,105,102,100,101,102,32,77,79,68,69,95,76,73,78,69,83,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,51,40,48,44,48,44,49,41,59,10,35,101,110,100,105,102,10,125,10,10,35,91,102,114,97,103,109,101,110,116,93,10,10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,52,32,100,115,116,95,99,111,108,111,114,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,100,115,116,95,99,111,108,111,114,32,61,32,118,101,99,52,40,49,44,49,44,48,44,48,41,59,10,125,10,0
+R"<!>(#[versions]
+
+lines = "#define MODE_LINES";
+
+#[vertex]
+
+#version 450
+
+#VERSION_DEFINES
+
+layout(location = 0) out vec3 uv_interp;
+
+void main() {
+
+#ifdef MODE_LINES
+	uv_interp = vec3(0,0,1);
+#endif
+}
+
+#[fragment]
+
+#version 450
+
+#VERSION_DEFINES
+
+#define M_PI 3.14159265359
+
+layout(location = 0) out vec4 dst_color;
+
+void main() {
+	dst_color = vec4(1,1,0,0);
+}
+)<!>"
 };
 #endif

tests/python_build/fixtures/rd_glsl/compute_expected_full.glsl

@@ -11,7 +11,19 @@ public:
 	ComputeShaderRD() {
 
 		static const char _compute_code[] = {
-10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,66,76,79,67,75,95,83,73,90,69,32,56,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,105,110,116,32,116,32,61,32,66,76,79,67,75,95,83,73,90,69,32,43,32,49,59,10,125,10,0
+R"<!>(
+#version 450
+
+#VERSION_DEFINES
+
+#define BLOCK_SIZE 8
+
+#define M_PI 3.14159265359
+
+void main() {
+	uint t = BLOCK_SIZE + 1;
+}
+)<!>"
 		};
 		setup(nullptr, nullptr, _compute_code, "ComputeShaderRD");
 	}

tests/python_build/fixtures/rd_glsl/vertex_fragment_expected_full.glsl

@@ -11,10 +11,33 @@ public:
 	VertexFragmentShaderRD() {
 
 		static const char _vertex_code[] = {
-10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,35,100,101,102,105,110,101,32,77,95,80,73,32,51,46,49,52,49,53,57,50,54,53,51,53,57,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,111,117,116,32,118,101,99,50,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,50,40,48,44,32,49,41,59,10,125,10,10,0
+R"<!>(
+#version 450
+
+#VERSION_DEFINES
+
+#define M_PI 3.14159265359
+
+layout(location = 0) out vec2 uv_interp;
+
+void main() {
+	uv_interp = vec2(0, 1);
+}
+
+)<!>"
 		};
 		static const char _fragment_code[] = {
-10,35,118,101,114,115,105,111,110,32,52,53,48,10,10,35,86,69,82,83,73,79,78,95,68,69,70,73,78,69,83,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,105,110,32,118,101,99,50,32,117,118,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,117,118,95,105,110,116,101,114,112,32,61,32,118,101,99,50,40,49,44,32,48,41,59,10,125,10,0
+R"<!>(
+#version 450
+
+#VERSION_DEFINES
+
+layout(location = 0) in vec2 uv_interp;
+
+void main() {
+	uv_interp = vec2(1, 0);
+}
+)<!>"
 		};
 		setup(_vertex_code, _fragment_code, nullptr, "VertexFragmentShaderRD");
 	}