Convert Python type hints in tools/make_rst.py

Since Python 3.5+ is required, we can use proper type hints based on PEP 484. Because we are working on contributor tooling we can assume 3.6+ and use PEP 526 for variable declarations as well.

doc/tools/make_rst.py

@@ -9,15 +9,13 @@ import re
 import sys
 import xml.etree.ElementTree as ET
 from collections import OrderedDict
+from typing import List, Dict, TextIO, Tuple, Optional, Any, Union
 
 # Import hardcoded version information from version.py
 root_directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
 sys.path.append(root_directory)  # Include the root directory
 import version
 
-# Uncomment to do type checks. I have it commented out so it works below Python 3.5
-# from typing import List, Dict, TextIO, Tuple, Iterable, Optional, DefaultDict, Any, Union
-
 # $DOCS_URL/path/to/page.html(#fragment-tag)
 GODOT_DOCS_PATTERN = re.compile(r"^\$DOCS_URL/(.*)\.html(#.*)?$")
 
@@ -61,126 +59,18 @@ BASE_STRINGS = [
     "This method doesn't need an instance to be called, so it can be called directly using the class name.",
     "This method describes a valid operator to use with this type as left-hand operand.",
 ]
-strings_l10n = {}
-
-STYLES = {}
-
+strings_l10n: Dict[str, str] = {}
 
-def print_error(error, state):  # type: (str, State) -> None
-    print("{}{}ERROR:{} {}{}".format(STYLES["red"], STYLES["bold"], STYLES["regular"], error, STYLES["reset"]))
-    state.num_errors += 1
-
-
-class TypeName:
-    def __init__(self, type_name, enum=None):  # type: (str, Optional[str]) -> None
-        self.type_name = type_name
-        self.enum = enum
-
-    def to_rst(self, state):  # type: ("State") -> str
-        if self.enum is not None:
-            return make_enum(self.enum, state)
-        elif self.type_name == "void":
-            return "void"
-        else:
-            return make_type(self.type_name, state)
-
-    @classmethod
-    def from_element(cls, element):  # type: (ET.Element) -> "TypeName"
-        return cls(element.attrib["type"], element.get("enum"))
-
-
-class PropertyDef:
-    def __init__(
-        self, name, type_name, setter, getter, text, default_value, overrides
-    ):  # type: (str, TypeName, Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> None
-        self.name = name
-        self.type_name = type_name
-        self.setter = setter
-        self.getter = getter
-        self.text = text
-        self.default_value = default_value
-        self.overrides = overrides
-
-
-class ParameterDef:
-    def __init__(self, name, type_name, default_value):  # type: (str, TypeName, Optional[str]) -> None
-        self.name = name
-        self.type_name = type_name
-        self.default_value = default_value
-
-
-class SignalDef:
-    def __init__(self, name, parameters, description):  # type: (str, List[ParameterDef], Optional[str]) -> None
-        self.name = name
-        self.parameters = parameters
-        self.description = description
-
-
-class MethodDef:
-    def __init__(
-        self, name, return_type, parameters, description, qualifiers
-    ):  # type: (str, TypeName, List[ParameterDef], Optional[str], Optional[str]) -> None
-        self.name = name
-        self.return_type = return_type
-        self.parameters = parameters
-        self.description = description
-        self.qualifiers = qualifiers
-
-
-class ConstantDef:
-    def __init__(self, name, value, text, bitfield):  # type: (str, str, Optional[str], Optional[bool]) -> None
-        self.name = name
-        self.value = value
-        self.text = text
-        self.is_bitfield = bitfield
-
-
-class EnumDef:
-    def __init__(self, name, bitfield):  # type: (str, Optional[bool]) -> None
-        self.name = name
-        self.values = OrderedDict()  # type: OrderedDict[str, ConstantDef]
-        self.is_bitfield = bitfield
-
-
-class ThemeItemDef:
-    def __init__(
-        self, name, type_name, data_name, text, default_value
-    ):  # type: (str, TypeName, str, Optional[str], Optional[str]) -> None
-        self.name = name
-        self.type_name = type_name
-        self.data_name = data_name
-        self.text = text
-        self.default_value = default_value
-
-
-class ClassDef:
-    def __init__(self, name):  # type: (str) -> None
-        self.name = name
-        self.constants = OrderedDict()  # type: OrderedDict[str, ConstantDef]
-        self.enums = OrderedDict()  # type: OrderedDict[str, EnumDef]
-        self.properties = OrderedDict()  # type: OrderedDict[str, PropertyDef]
-        self.constructors = OrderedDict()  # type: OrderedDict[str, List[MethodDef]]
-        self.methods = OrderedDict()  # type: OrderedDict[str, List[MethodDef]]
-        self.operators = OrderedDict()  # type: OrderedDict[str, List[MethodDef]]
-        self.signals = OrderedDict()  # type: OrderedDict[str, SignalDef]
-        self.annotations = OrderedDict()  # type: OrderedDict[str, List[MethodDef]]
-        self.theme_items = OrderedDict()  # type: OrderedDict[str, ThemeItemDef]
-        self.inherits = None  # type: Optional[str]
-        self.brief_description = None  # type: Optional[str]
-        self.description = None  # type: Optional[str]
-        self.tutorials = []  # type: List[Tuple[str, str]]
-
-        # Used to match the class with XML source for output filtering purposes.
-        self.filepath = ""  # type: str
+STYLES: Dict[str, str] = {}
 
 
 class State:
-    def __init__(self):  # type: () -> None
+    def __init__(self) -> None:
         self.num_errors = 0
-        self.classes = OrderedDict()  # type: OrderedDict[str, ClassDef]
-        self.current_class = ""  # type: str
+        self.classes: OrderedDict[str, ClassDef] = OrderedDict()
+        self.current_class: str = ""
 
-    def parse_class(self, class_root, filepath):  # type: (ET.Element, str) -> None
+    def parse_class(self, class_root: ET.Element, filepath: str) -> None:
         class_name = class_root.attrib["name"]
 
         class_def = ClassDef(class_name)
@@ -233,7 +123,6 @@ class State:
                 return_element = constructor.find("return")
                 if return_element is not None:
                     return_type = TypeName.from_element(return_element)
-
                 else:
                     return_type = TypeName("void")
 
@@ -314,7 +203,7 @@ class State:
                 constant_name = constant.attrib["name"]
                 value = constant.attrib["value"]
                 enum = constant.get("enum")
-                is_bitfield = constant.get("is_bitfield") or False
+                is_bitfield = constant.get("is_bitfield") == "true"
                 constant_def = ConstantDef(constant_name, value, constant.text, is_bitfield)
                 if enum is None:
                     if constant_name in class_def.constants:
@@ -348,6 +237,7 @@ class State:
                 if desc_element is not None:
                     annotation_desc = desc_element.text
 
+                return_type = TypeName("void")
                 annotation_def = MethodDef(annotation_name, return_type, params, annotation_desc, qualifiers)
                 if annotation_name not in class_def.annotations:
                     class_def.annotations[annotation_name] = []
@@ -413,13 +303,133 @@ class State:
                 if link.text is not None:
                     class_def.tutorials.append((link.text.strip(), link.get("title", "")))
 
-    def sort_classes(self):  # type: () -> None
+    def sort_classes(self) -> None:
         self.classes = OrderedDict(sorted(self.classes.items(), key=lambda t: t[0]))
 
 
-def parse_arguments(root):  # type: (ET.Element) -> List[ParameterDef]
+class TypeName:
+    def __init__(self, type_name: str, enum: Optional[str] = None) -> None:
+        self.type_name = type_name
+        self.enum = enum
+
+    def to_rst(self, state: State) -> str:
+        if self.enum is not None:
+            return make_enum(self.enum, state)
+        elif self.type_name == "void":
+            return "void"
+        else:
+            return make_type(self.type_name, state)
+
+    @classmethod
+    def from_element(cls, element: ET.Element) -> "TypeName":
+        return cls(element.attrib["type"], element.get("enum"))
+
+
+class PropertyDef:
+    def __init__(
+        self,
+        name: str,
+        type_name: TypeName,
+        setter: Optional[str],
+        getter: Optional[str],
+        text: Optional[str],
+        default_value: Optional[str],
+        overrides: Optional[str],
+    ) -> None:
+        self.name = name
+        self.type_name = type_name
+        self.setter = setter
+        self.getter = getter
+        self.text = text
+        self.default_value = default_value
+        self.overrides = overrides
+
+
+class ParameterDef:
+    def __init__(self, name: str, type_name: TypeName, default_value: Optional[str]) -> None:
+        self.name = name
+        self.type_name = type_name
+        self.default_value = default_value
+
+
+class SignalDef:
+    def __init__(self, name: str, parameters: List[ParameterDef], description: Optional[str]) -> None:
+        self.name = name
+        self.parameters = parameters
+        self.description = description
+
+
+class MethodDef:
+    def __init__(
+        self,
+        name: str,
+        return_type: TypeName,
+        parameters: List[ParameterDef],
+        description: Optional[str],
+        qualifiers: Optional[str],
+    ) -> None:
+        self.name = name
+        self.return_type = return_type
+        self.parameters = parameters
+        self.description = description
+        self.qualifiers = qualifiers
+
+
+class ConstantDef:
+    def __init__(self, name: str, value: str, text: Optional[str], bitfield: bool) -> None:
+        self.name = name
+        self.value = value
+        self.text = text
+        self.is_bitfield = bitfield
+
+
+class EnumDef:
+    def __init__(self, name: str, bitfield: bool) -> None:
+        self.name = name
+        self.values: OrderedDict[str, ConstantDef] = OrderedDict()
+        self.is_bitfield = bitfield
+
+
+class ThemeItemDef:
+    def __init__(
+        self, name: str, type_name: TypeName, data_name: str, text: Optional[str], default_value: Optional[str]
+    ) -> None:
+        self.name = name
+        self.type_name = type_name
+        self.data_name = data_name
+        self.text = text
+        self.default_value = default_value
+
+
+class ClassDef:
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.constants: OrderedDict[str, ConstantDef] = OrderedDict()
+        self.enums: OrderedDict[str, EnumDef] = OrderedDict()
+        self.properties: OrderedDict[str, PropertyDef] = OrderedDict()
+        self.constructors: OrderedDict[str, List[MethodDef]] = OrderedDict()
+        self.methods: OrderedDict[str, List[MethodDef]] = OrderedDict()
+        self.operators: OrderedDict[str, List[MethodDef]] = OrderedDict()
+        self.signals: OrderedDict[str, SignalDef] = OrderedDict()
+        self.annotations: OrderedDict[str, List[MethodDef]] = OrderedDict()
+        self.theme_items: OrderedDict[str, ThemeItemDef] = OrderedDict()
+        self.inherits: Optional[str] = None
+        self.brief_description: Optional[str] = None
+        self.description: Optional[str] = None
+        self.tutorials: List[Tuple[str, str]] = []
+
+        # Used to match the class with XML source for output filtering purposes.
+        self.filepath: str = ""
+
+
+def print_error(error: str, state: State) -> None:
+    print("{}{}ERROR:{} {}{}".format(STYLES["red"], STYLES["bold"], STYLES["regular"], error, STYLES["reset"]))
+    state.num_errors += 1
+
+
+def parse_arguments(root: ET.Element) -> List[ParameterDef]:
     param_elements = root.findall("argument")
-    params = [None] * len(param_elements)  # type: Any
+    params: Any = [None] * len(param_elements)
     for param_element in param_elements:
         param_name = param_element.attrib["name"]
         index = int(param_element.attrib["index"])
@@ -428,16 +438,16 @@ def parse_arguments(root):  # type: (ET.Element) -> List[ParameterDef]
 
         params[index] = ParameterDef(param_name, type_name, default)
 
-    cast = params  # type: List[ParameterDef]
+    cast: List[ParameterDef] = params
 
     return cast
 
 
-def main():  # type: () -> None
+def main() -> None:
     # Enable ANSI escape code support on Windows 10 and later (for colored console output).
     # <https://bugs.python.org/issue29059>
     if platform.system().lower() == "windows":
-        from ctypes import windll, c_int, byref
+        from ctypes import windll, c_int, byref  # type: ignore
 
         stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
         mode = c_int(0)
@@ -491,7 +501,7 @@ def main():  # type: () -> None
 
     print("Checking for errors in the XML class reference...")
 
-    file_list = []  # type: List[str]
+    file_list: List[str] = []
 
     for path in args.path:
         # Cut off trailing slashes so os.path.basename doesn't choke.
@@ -515,7 +525,7 @@ def main():  # type: () -> None
 
             file_list.append(path)
 
-    classes = {}  # type: Dict[str, ET.Element]
+    classes: Dict[str, Tuple[ET.Element, str]] = {}
     state = State()
 
     for cur_file in file_list:
@@ -576,7 +586,7 @@ def main():  # type: () -> None
         exit(1)
 
 
-def translate(string):  # type: (str) -> str
+def translate(string: str) -> str:
     """Translate a string based on translations sourced from `doc/translations/*.po`
     for a language if defined via the --lang command line argument.
     Returns the original string if no translation exists.
@@ -584,7 +594,7 @@ def translate(string):  # type: (str) -> str
     return strings_l10n.get(string, string)
 
 
-def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, State, bool, str) -> None
+def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: str) -> None:
     class_name = class_def.name
 
     if dry_run:
@@ -635,8 +645,8 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
                 break
         f.write("\n\n")
 
-    # Descendents
-    inherited = []
+    # Descendants
+    inherited: List[str] = []
     for c in state.classes.values():
         if c.inherits and c.inherits.strip() == class_name:
             inherited.append(c.name)
@@ -667,7 +677,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
     # Properties overview
     if len(class_def.properties) > 0:
         f.write(make_heading("Properties", "-"))
-        ml = []  # type: List[Tuple[str, str, str]]
+        ml: List[Tuple[Optional[str], ...]] = []
         for property_def in class_def.properties.values():
             type_rst = property_def.type_name.to_rst(state)
             default = property_def.default_value
@@ -683,7 +693,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
     # Constructors, Methods, Operators overview
     if len(class_def.constructors) > 0:
         f.write(make_heading("Constructors", "-"))
-        ml = []
+        ml: List[Tuple[Optional[str], ...]] = []
         for method_list in class_def.constructors.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "constructor", state))
@@ -691,7 +701,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
 
     if len(class_def.methods) > 0:
         f.write(make_heading("Methods", "-"))
-        ml = []
+        ml: List[Tuple[Optional[str], ...]] = []
         for method_list in class_def.methods.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "method", state))
@@ -699,7 +709,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
 
     if len(class_def.operators) > 0:
         f.write(make_heading("Operators", "-"))
-        ml = []
+        ml: List[Tuple[Optional[str], ...]] = []
         for method_list in class_def.operators.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "operator", state))
@@ -708,7 +718,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
     # Theme properties
     if len(class_def.theme_items) > 0:
         f.write(make_heading("Theme Properties", "-"))
-        pl = []
+        pl: List[Tuple[Optional[str], ...]] = []
         for theme_item_def in class_def.theme_items.values():
             ref = ":ref:`{0}<class_{2}_theme_{1}_{0}>`".format(
                 theme_item_def.name, theme_item_def.data_name, class_name
@@ -816,7 +826,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
             f.write(".. _class_{}_property_{}:\n\n".format(class_name, property_def.name))
             f.write("- {} **{}**\n\n".format(property_def.type_name.to_rst(state), property_def.name))
 
-            info = []
+            info: List[Tuple[Optional[str], ...]] = []
             # Not using translate() for now as it breaks table formatting.
             if property_def.default_value is not None:
                 info.append(("*" + "Default" + "*", property_def.default_value))
@@ -926,7 +936,7 @@ def make_rst_class(class_def, state, dry_run, output_dir):  # type: (ClassDef, S
     f.write(make_footer())
 
 
-def escape_rst(text, until_pos=-1):  # type: (str, int) -> str
+def escape_rst(text: str, until_pos: int = -1) -> str:
     # Escape \ character, otherwise it ends up as an escape character in rst
     pos = 0
     while True:
@@ -960,7 +970,7 @@ def escape_rst(text, until_pos=-1):  # type: (str, int) -> str
     return text
 
 
-def format_codeblock(code_type, post_text, indent_level, state):  # types: str, str, int, state
+def format_codeblock(code_type: str, post_text: str, indent_level: int, state: State) -> Union[Tuple[str, int], None]:
     end_pos = post_text.find("[/" + code_type + "]")
     if end_pos == -1:
         print_error("{}.xml: [" + code_type + "] without a closing tag.".format(state.current_class), state)
@@ -994,10 +1004,10 @@ def format_codeblock(code_type, post_text, indent_level, state):  # types: str,
         else:
             code_text = code_text[:code_pos] + "\n    " + code_text[code_pos + to_skip + 1 :]
             code_pos += 5 - to_skip
-    return ["\n[" + code_type + "]" + code_text + post_text, len("\n[" + code_type + "]" + code_text)]
+    return ("\n[" + code_type + "]" + code_text + post_text, len("\n[" + code_type + "]" + code_text))
 
 
-def rstize_text(text, state):  # type: (str, State) -> str
+def rstize_text(text: str, state: State) -> str:
     # Linebreak + tabs in the XML should become two line breaks unless in a "codeblock"
     pos = 0
     while True:
@@ -1037,7 +1047,6 @@ def rstize_text(text, state):  # type: (str, State) -> str
     inside_code = False
     pos = 0
     tag_depth = 0
-    previous_pos = 0
     while True:
         pos = text.find("[", pos)
         if pos == -1:
@@ -1204,7 +1213,6 @@ def rstize_text(text, state):  # type: (str, State) -> str
 
                 text = pre_text + tag_text + post_text
                 pos = len(pre_text) + len(tag_text)
-                previous_pos = pos
                 continue
             elif cmd == "center":
                 tag_depth += 1
@@ -1310,7 +1318,6 @@ def rstize_text(text, state):  # type: (str, State) -> str
 
         text = pre_text + tag_text + post_text
         pos = len(pre_text) + len(tag_text)
-        previous_pos = pos
 
     if tag_depth > 0:
         print_error(
@@ -1320,7 +1327,7 @@ def rstize_text(text, state):  # type: (str, State) -> str
     return text
 
 
-def format_table(f, data, remove_empty_columns=False):  # type: (TextIO, Iterable[Tuple[str, ...]], bool) -> None
+def format_table(f: TextIO, data: List[Tuple[Optional[str], ...]], remove_empty_columns: bool = False) -> None:
     if len(data) == 0:
         return
 
@@ -1351,7 +1358,7 @@ def format_table(f, data, remove_empty_columns=False):  # type: (TextIO, Iterabl
     f.write("\n")
 
 
-def make_type(klass, state):  # type: (str, State) -> str
+def make_type(klass: str, state: State) -> str:
     if klass.find("*") != -1:  # Pointer, ignore
         return klass
     link_type = klass
@@ -1363,7 +1370,7 @@ def make_type(klass, state):  # type: (str, State) -> str
     return klass
 
 
-def make_enum(t, state):  # type: (str, State) -> str
+def make_enum(t: str, state: State) -> str:
     p = t.find(".")
     if p >= 0:
         c = t[0:p]
@@ -1389,16 +1396,17 @@ def make_enum(t, state):  # type: (str, State) -> str
 
 
 def make_method_signature(
-    class_def, method_def, ref_type, state
-):  # type: (ClassDef, Union[MethodDef, SignalDef], str, State) -> Tuple[str, str]
+    class_def: ClassDef, method_def: Union[MethodDef, SignalDef], ref_type: str, state: State
+) -> Tuple[str, str]:
     ret_type = " "
 
-    if isinstance(method_def, MethodDef):
+    is_method_def = isinstance(method_def, MethodDef)
+    if is_method_def:
         ret_type = method_def.return_type.to_rst(state)
 
     out = ""
 
-    if ref_type != "":
+    if is_method_def and ref_type != "":
         if ref_type == "operator":
             out += ":ref:`{0}<class_{1}_{2}_{3}_{4}>` ".format(
                 method_def.name.replace("<", "\\<"),  # So operator "<" gets correctly displayed.
@@ -1441,7 +1449,7 @@ def make_method_signature(
     return ret_type, out
 
 
-def make_heading(title, underline, l10n=True):  # type: (str, str, bool) -> str
+def make_heading(title: str, underline: str, l10n: bool = True) -> str:
     if l10n:
         new_title = translate(title)
         if new_title != title:
@@ -1450,7 +1458,7 @@ def make_heading(title, underline, l10n=True):  # type: (str, str, bool) -> str
     return title + "\n" + (underline * len(title)) + "\n\n"
 
 
-def make_footer():  # type: () -> str
+def make_footer() -> str:
     # Generate reusable abbreviation substitutions.
     # This way, we avoid bloating the generated rST with duplicate abbreviations.
     # fmt: off
@@ -1465,7 +1473,7 @@ def make_footer():  # type: () -> str
     # fmt: on
 
 
-def make_link(url, title):  # type: (str, str) -> str
+def make_link(url: str, title: str) -> str:
     match = GODOT_DOCS_PATTERN.search(url)
     if match:
         groups = match.groups()
@@ -1482,15 +1490,15 @@ def make_link(url, title):  # type: (str, str) -> str
             if title != "":
                 return ":doc:`" + title + " <../" + groups[0] + ">`"
             return ":doc:`../" + groups[0] + "`"
-    else:
-        # External link, for example:
-        # `http://enet.bespin.org/usergroup0.html`
-        if title != "":
-            return "`" + title + " <" + url + ">`__"
-        return "`" + url + " <" + url + ">`__"
+
+    # External link, for example:
+    # `http://enet.bespin.org/usergroup0.html`
+    if title != "":
+        return "`" + title + " <" + url + ">`__"
+    return "`" + url + " <" + url + ">`__"
 
 
-def sanitize_operator_name(dirty_name, state):  # type: (str, State) -> str
+def sanitize_operator_name(dirty_name: str, state: State) -> str:
     clear_name = dirty_name.replace("operator ", "")
 
     if clear_name == "!=":
@@ -1548,7 +1556,7 @@ def sanitize_operator_name(dirty_name, state):  # type: (str, State) -> str
     return clear_name
 
 
-def indent_bullets(text):  # type: (str) -> str
+def indent_bullets(text: str) -> str:
     # Take the text and check each line for a bullet point represented by "-".
     # Where found, indent the given line by a further "\t".
     # Used to properly indent bullet points contained in the description for enum values.