Merge pull request #37294 from akien-mga/scons-drop-python2

SCons: Drop support for Python 2

SConstruct

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-EnsureSConsVersion(0, 98, 1)
+EnsureSConsVersion(3, 0, 0)
+EnsurePythonVersion(3, 5)
 
 # System
 import glob
@@ -13,7 +14,7 @@ import methods
 import gles_builders
 from platform_methods import run_in_subprocess
 
-# scan possible build platforms
+# Scan possible build platforms
 
 platform_list = []  # list of platforms
 platform_opts = {}  # options for each platform

compat.py

@@ -1,68 +0,0 @@
-import sys
-
-if sys.version_info < (3,):
-    def isbasestring(s):
-        return isinstance(s, basestring)
-    def open_utf8(filename, mode):
-        return open(filename, mode)
-    def byte_to_str(x):
-        return str(ord(x))
-    import cStringIO
-    def StringIO():
-        return cStringIO.StringIO()
-    def encode_utf8(x):
-        return x
-    def decode_utf8(x):
-        return x
-    def iteritems(d):
-        return d.iteritems()
-    def itervalues(d):
-        return d.itervalues()
-    def escape_string(s):
-        if isinstance(s, unicode):
-            s = s.encode('ascii')
-        result = ''
-        for c in s:
-            if not (32 <= ord(c) < 127) or c in ('\\', '"'):
-                result += '\\%03o' % ord(c)
-            else:
-                result += c
-        return result
-
-else:
-    def isbasestring(s):
-        return isinstance(s, (str, bytes))
-    def open_utf8(filename, mode):
-        return open(filename, mode, encoding="utf-8")
-    def byte_to_str(x):
-        return str(x)
-    import io
-    def StringIO():
-        return io.StringIO()
-    import codecs
-    def encode_utf8(x):
-        return codecs.utf_8_encode(x)[0]
-    def decode_utf8(x):
-        return codecs.utf_8_decode(x)[0]
-    def iteritems(d):
-        return iter(d.items())
-    def itervalues(d):
-        return iter(d.values())
-    def charcode_to_c_escapes(c):
-        rev_result = []
-        while c >= 256:
-            c, low = (c // 256, c % 256)
-            rev_result.append('\\%03o' % low)
-        rev_result.append('\\%03o' % c)
-        return ''.join(reversed(rev_result))
-    def escape_string(s):
-        result = ''
-        if isinstance(s, str):
-            s = s.encode('utf-8')
-        for c in s:
-            if not(32 <= c < 127) or c in (ord('\\'), ord('"')):
-                result += charcode_to_c_escapes(c)
-            else:
-                result += chr(c)
-        return result
-

core/core_builders.py

@@ -4,15 +4,33 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki
 """
 
 from platform_methods import subprocess_main
-from compat import iteritems, itervalues, open_utf8, escape_string, byte_to_str
 
 
-def make_certs_header(target, source, env):
+def escape_string(s):
+    def charcode_to_c_escapes(c):
+        rev_result = []
+        while c >= 256:
+            c, low = (c // 256, c % 256)
+            rev_result.append('\\%03o' % low)
+        rev_result.append('\\%03o' % c)
+        return ''.join(reversed(rev_result))
+
+    result = ''
+    if isinstance(s, str):
+        s = s.encode('utf-8')
+    for c in s:
+        if not(32 <= c < 127) or c in (ord('\\'), ord('"')):
+            result += charcode_to_c_escapes(c)
+        else:
+            result += chr(c)
+    return result
 
+
+def make_certs_header(target, source, env):
     src = source[0]
     dst = target[0]
     f = open(src, "rb")
-    g = open_utf8(dst, "w")
+    g = open(dst, "w", encoding="utf-8")
     buf = f.read()
     decomp_size = len(buf)
     import zlib
@@ -32,7 +50,7 @@ def make_certs_header(target, source, env):
         g.write("static const int _certs_uncompressed_size = " + str(decomp_size) + ";\n")
         g.write("static const unsigned char _certs_compressed[] = {\n")
         for i in range(len(buf)):
-            g.write("\t" + byte_to_str(buf[i]) + ",\n")
+            g.write("\t" + str(buf[i]) + ",\n")
         g.write("};\n")
     g.write("#endif // CERTS_COMPRESSED_GEN_H")
 
@@ -46,8 +64,8 @@ def make_authors_header(target, source, env):
 
     src = source[0]
     dst = target[0]
-    f = open_utf8(src, "r")
-    g = open_utf8(dst, "w")
+    f = open(src, "r", encoding="utf-8")
+    g = open(dst, "w", encoding="utf-8")
 
     g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
     g.write("#ifndef AUTHORS_GEN_H\n")
@@ -92,8 +110,8 @@ def make_donors_header(target, source, env):
 
     src = source[0]
     dst = target[0]
-    f = open_utf8(src, "r")
-    g = open_utf8(dst, "w")
+    f = open(src, "r", encoding="utf-8")
+    g = open(dst, "w", encoding="utf-8")
 
     g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
     g.write("#ifndef DONORS_GEN_H\n")
@@ -163,7 +181,7 @@ def make_license_header(target, source, env):
     projects = OrderedDict()
     license_list = []
 
-    with open_utf8(src_copyright, "r") as copyright_file:
+    with open(src_copyright, "r", encoding="utf-8") as copyright_file:
         reader = LicenseReader(copyright_file)
         part = {}
         while reader.current:
@@ -183,21 +201,21 @@ def make_license_header(target, source, env):
                 reader.next_line()
 
     data_list = []
-    for project in itervalues(projects):
+    for project in iter(projects.values()):
         for part in project:
             part["file_index"] = len(data_list)
             data_list += part["Files"]
             part["copyright_index"] = len(data_list)
             data_list += part["Copyright"]
 
-    with open_utf8(dst, "w") as f:
+    with open(dst, "w", encoding="utf-8") as f:
 
         f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
         f.write("#ifndef LICENSE_GEN_H\n")
         f.write("#define LICENSE_GEN_H\n")
         f.write("const char *const GODOT_LICENSE_TEXT =")
 
-        with open_utf8(src_license, "r") as license_file:
+        with open(src_license, "r", encoding="utf-8") as license_file:
             for line in license_file:
                 escaped_string = escape_string(line.strip())
                 f.write("\n\t\t\"" + escaped_string + "\\n\"")
@@ -225,7 +243,7 @@ def make_license_header(target, source, env):
         f.write("const ComponentCopyrightPart COPYRIGHT_PROJECT_PARTS[] = {\n")
         part_index = 0
         part_indexes = {}
-        for project_name, project in iteritems(projects):
+        for project_name, project in iter(projects.items()):
             part_indexes[project_name] = part_index
             for part in project:
                 f.write("\t{ \"" + escape_string(part["License"][0]) + "\", "
@@ -239,7 +257,7 @@ def make_license_header(target, source, env):
         f.write("const int COPYRIGHT_INFO_COUNT = " + str(len(projects)) + ";\n")
 
         f.write("const ComponentCopyright COPYRIGHT_INFO[] = {\n")
-        for project_name, project in iteritems(projects):
+        for project_name, project in iter(projects.items()):
             f.write("\t{ \"" + escape_string(project_name) + "\", "
                     + "&COPYRIGHT_PROJECT_PARTS[" + str(part_indexes[project_name]) + "], "
                     + str(len(project)) + " },\n")

doc/Makefile

@@ -13,13 +13,6 @@ doxygen:
 	mkdir -p $(OUTPUTDIR)/doxygen
 	doxygen Doxyfile
 
-markdown:
-	rm -rf $(OUTPUTDIR)/markdown
-	mkdir -p $(OUTPUTDIR)/markdown
-	pushd $(OUTPUTDIR)/markdown
-	python2 $(TOOLSDIR)/makemd.py $(CLASSES)
-	popd
-
 rst:
 	rm -rf $(OUTPUTDIR)/rst
 	mkdir -p $(OUTPUTDIR)/rst

editor/SCsub

@@ -7,13 +7,12 @@ env.editor_sources = []
 import os
 import os.path
 from platform_methods import run_in_subprocess
-from compat import open_utf8
 import editor_builders
 
 
 def _make_doc_data_class_path(to_path):
     # NOTE: It is safe to generate this file here, since this is still executed serially
-    g = open_utf8(os.path.join(to_path, "doc_data_class_path.gen.h"), "w")
+    g = open(os.path.join(to_path, "doc_data_class_path.gen.h"), "w", encoding="utf-8")
     g.write("static const int _doc_data_class_path_count = " + str(len(env.doc_class_path)) + ";\n")
     g.write("struct _DocDataClassPath { const char* name; const char* path; };\n")
 
@@ -37,7 +36,7 @@ if env['tools']:
     reg_exporters += '}\n'
 
     # NOTE: It is safe to generate this file here, since this is still executed serially
-    with open_utf8("register_exporters.gen.cpp", "w") as f:
+    with open("register_exporters.gen.cpp", "w", encoding="utf-8") as f:
         f.write(reg_exporters_inc)
         f.write(reg_exporters)
 

editor/editor_builders.py

@@ -6,24 +6,23 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki
 import os
 import os.path
 from platform_methods import subprocess_main
-from compat import encode_utf8, byte_to_str, open_utf8
 
 
 def make_doc_header(target, source, env):
 
     dst = target[0]
-    g = open_utf8(dst, "w")
+    g = open(dst, "w", encoding="utf-8")
     buf = ""
     docbegin = ""
     docend = ""
     for src in source:
         if not src.endswith(".xml"):
             continue
-        with open_utf8(src, "r") as f:
+        with open(src, "r", encoding="utf-8") as f:
             content = f.read()
         buf += content
 
-    buf = encode_utf8(docbegin + buf + docend)
+    buf = (docbegin + buf + docend).encode("utf-8")
     decomp_size = len(buf)
     import zlib
     buf = zlib.compress(buf)
@@ -35,7 +34,7 @@ def make_doc_header(target, source, env):
     g.write("static const int _doc_data_uncompressed_size = " + str(decomp_size) + ";\n")
     g.write("static const unsigned char _doc_data_compressed[] = {\n")
     for i in range(len(buf)):
-        g.write("\t" + byte_to_str(buf[i]) + ",\n")
+        g.write("\t" + str(buf[i]) + ",\n")
     g.write("};\n")
 
     g.write("#endif")
@@ -47,7 +46,7 @@ def make_fonts_header(target, source, env):
 
     dst = target[0]
 
-    g = open_utf8(dst, "w")
+    g = open(dst, "w", encoding="utf-8")
 
     g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
     g.write("#ifndef _EDITOR_FONTS_H\n")
@@ -64,7 +63,7 @@ def make_fonts_header(target, source, env):
         g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n")
         g.write("static const unsigned char _font_" + name + "[] = {\n")
         for j in range(len(buf)):
-            g.write("\t" + byte_to_str(buf[j]) + ",\n")
+            g.write("\t" + str(buf[j]) + ",\n")
 
         g.write("};\n")
 
@@ -77,7 +76,7 @@ def make_translations_header(target, source, env, category):
 
     dst = target[0]
 
-    g = open_utf8(dst, "w")
+    g = open(dst, "w", encoding="utf-8")
 
     g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
     g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper()))
@@ -98,7 +97,7 @@ def make_translations_header(target, source, env, category):
 
         g.write("static const unsigned char _{}_translation_{}_compressed[] = {{\n".format(category, name))
         for j in range(len(buf)):
-            g.write("\t" + byte_to_str(buf[j]) + ",\n")
+            g.write("\t" + str(buf[j]) + ",\n")
 
         g.write("};\n")
 

editor/icons/editor_icons_builders.py

@@ -3,9 +3,10 @@
 All such functions are invoked in a subprocess on Windows to prevent build flakiness.
 
 """
+
 import os
+from io import StringIO
 from platform_methods import subprocess_main
-from compat import StringIO
 
 
 def make_editor_icons_action(target, source, env):

main/main_builders.py

@@ -4,7 +4,6 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki
 
 """
 from platform_methods import subprocess_main
-from compat import byte_to_str
 from collections import OrderedDict
 
 
@@ -22,7 +21,7 @@ def make_splash(target, source, env):
         g.write('static const Color boot_splash_bg_color = Color(0.14, 0.14, 0.14);\n')
         g.write("static const unsigned char boot_splash_png[] = {\n")
         for i in range(len(buf)):
-            g.write(byte_to_str(buf[i]) + ",\n")
+            g.write(str(buf[i]) + ",\n")
         g.write("};\n")
         g.write("#endif")
 
@@ -41,7 +40,7 @@ def make_splash_editor(target, source, env):
         g.write('static const Color boot_splash_editor_bg_color = Color(0.14, 0.14, 0.14);\n')
         g.write("static const unsigned char boot_splash_editor_png[] = {\n")
         for i in range(len(buf)):
-            g.write(byte_to_str(buf[i]) + ",\n")
+            g.write(str(buf[i]) + ",\n")
         g.write("};\n")
         g.write("#endif")
 
@@ -59,7 +58,7 @@ def make_app_icon(target, source, env):
         g.write("#define APP_ICON_H\n")
         g.write("static const unsigned char app_icon_png[] = {\n")
         for i in range(len(buf)):
-            g.write(byte_to_str(buf[i]) + ",\n")
+            g.write(str(buf[i]) + ",\n")
         g.write("};\n")
         g.write("#endif")
 

methods.py

@@ -2,12 +2,11 @@ import os
 import re
 import glob
 import subprocess
-from compat import iteritems, isbasestring, decode_utf8
 
 
 def add_source_files(self, sources, files, warn_duplicates=True):
     # Convert string to list of absolute paths (including expanding wildcard)
-    if isbasestring(files):
+    if isinstance(files, (str, bytes)):
         # Keep SCons project-absolute path as they are (no wildcard support)
         if files.startswith('#'):
             if '*' in files:
@@ -240,7 +239,7 @@ def use_windows_spawn_fix(self, platform=None):
         cmdline = cmd + " " + newargs
 
         rv = 0
-        env = {str(key): str(value) for key, value in iteritems(env)}
+        env = {str(key): str(value) for key, value in iter(env.items())}
         if len(cmdline) > 32000 and cmd.endswith("ar"):
             cmdline = cmd + " " + args[1] + " " + args[2] + " "
             for i in range(3, len(args)):
@@ -530,7 +529,7 @@ def detect_darwin_sdk_path(platform, env):
 
     if not env[var_name]:
         try:
-            sdk_path = decode_utf8(subprocess.check_output(['xcrun', '--sdk', sdk_name, '--show-sdk-path']).strip())
+            sdk_path = subprocess.check_output(['xcrun', '--sdk', sdk_name, '--show-sdk-path']).strip().decode("utf-8")
             if sdk_path:
                 env[var_name] = sdk_path
         except (subprocess.CalledProcessError, OSError):
@@ -540,7 +539,7 @@ def detect_darwin_sdk_path(platform, env):
 def is_vanilla_clang(env):
     if not using_clang(env):
         return False
-    version = decode_utf8(subprocess.check_output([env['CXX'], '--version']).strip())
+    version = subprocess.check_output([env['CXX'], '--version']).strip().decode("utf-8")
     return not version.startswith("Apple")
 
 
@@ -553,7 +552,7 @@ def get_compiler_version(env):
         # Not using -dumpversion as some GCC distros only return major, and
         # Clang used to return hardcoded 4.2.1: # https://reviews.llvm.org/D56803
         try:
-            version = decode_utf8(subprocess.check_output([env.subst(env['CXX']), '--version']).strip())
+            version = subprocess.check_output([env.subst(env['CXX']), '--version']).strip().decode("utf-8")
         except (subprocess.CalledProcessError, OSError):
             print("Couldn't parse CXX environment variable to infer compiler version.")
             return None

modules/freetype/SCsub

@@ -3,8 +3,6 @@
 Import('env')
 Import('env_modules')
 
-from compat import isbasestring
-
 env_freetype = env_modules.Clone()
 
 # Thirdparty source files
@@ -93,7 +91,7 @@ if env['builtin_freetype']:
     # and then plain strings for system library. We insert between the two.
     inserted = False
     for idx, linklib in enumerate(env["LIBS"]):
-        if isbasestring(linklib): # first system lib such as "X11", otherwise SCons lib object
+        if isinstance(linklib, (str, bytes)): # first system lib such as "X11", otherwise SCons lib object
             env["LIBS"].insert(idx, lib)
             inserted = True
             break

modules/mono/build_scripts/make_android_mono_config.py

@@ -1,7 +1,6 @@
 
 def generate_compressed_config(config_src, output_dir):
     import os.path
-    from compat import byte_to_str
 
     # Source file
     with open(os.path.join(output_dir, 'android_mono_config.gen.cpp'), 'w') as cpp:
@@ -16,7 +15,7 @@ def generate_compressed_config(config_src, output_dir):
             for i, buf_idx in enumerate(range(compr_size)):
                 if i > 0:
                     bytes_seq_str += ', '
-                bytes_seq_str += byte_to_str(buf[buf_idx])
+                bytes_seq_str += str(buf[buf_idx])
 
             cpp.write('''/* THIS FILE IS GENERATED DO NOT EDIT */
 #include "android_mono_config.h"

modules/mono/build_scripts/mono_reg_utils.py

@@ -1,14 +1,9 @@
 import os
 import platform
 
-from compat import decode_utf8
-
 if os.name == 'nt':
     import sys
-    if sys.version_info < (3,):
-        import _winreg as winreg
-    else:
-        import winreg
+    import winreg
 
 
 def _reg_open_key(key, subkey):
@@ -81,7 +76,7 @@ def find_msbuild_tools_path_reg():
         lines = subprocess.check_output([vswhere] + vswhere_args).splitlines()
 
         for line in lines:
-            parts = decode_utf8(line).split(':', 1)
+            parts = line.decode("utf-8").split(':', 1)
 
             if len(parts) < 2 or parts[0] != 'installationPath':
                 continue

platform/SCsub

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from compat import open_utf8
-
 Import('env')
 
 env.platform_sources = []
@@ -21,7 +19,7 @@ reg_apis += '}\n\n'
 unreg_apis += '}\n'
 
 # NOTE: It is safe to generate this file here, since this is still execute serially
-with open_utf8('register_platform_apis.gen.cpp', 'w') as f:
+with open('register_platform_apis.gen.cpp', 'w', encoding="utf-8") as f:
     f.write(reg_apis_inc)
     f.write(reg_apis)
     f.write(unreg_apis)

platform_methods.py

@@ -7,10 +7,7 @@ import subprocess
 
 # NOTE: The multiprocessing module is not compatible with SCons due to conflict on cPickle
 
-if sys.version_info[0] < 3:
-    JSON_SERIALIZABLE_TYPES = (bool, int, long, float, basestring)
-else:
-    JSON_SERIALIZABLE_TYPES = (bool, int, float, str)
+JSON_SERIALIZABLE_TYPES = (bool, int, float, str)
 
 
 def run_in_subprocess(builder_function):