Appease our great search engine overlords

This hopefully fixes all translated pages showing up in search engine results, and makes the STABLE version of each page canonical.
In turn, this allows us to re-enable indexing of the version-specific pages (see robots.txt changes), as search engines should prefer the canonical (stable) version, and only show the other versions if no canonical (stable) version exists (i.e. because that feature is only in latest, or was removed in stable).

It adds proper canonical links for all generated pages, and fixes the existing links between the various translations of a page by both ensuring the pages links to itself with the proper language tag, and by properly linking to the full path of other translated versions.

(cherry picked from commits:
- fd5f6f49097ab916ff3f5daa433c6e57d0a59716,
- 263ff562516c98fa6113c920ea7281e21d2d9f79,
- e21df0671fe5a5aeb0e6253084dd842bad6114da,
- 24781e377baad9375aaa7c7175e0bedb5c9c0414,
- 66d185d5d2d51cfd8c28716930d1ca04ff51090b,
- 3c79f3e3218bff53a6303e7b85ebe2089c09c4b6)

Co-authored-by: Rémi Verschelde <[email protected]>

_templates/layout.html

@@ -1,14 +1,14 @@
-{% extends "!layout.html" %}
-{% block linktags %}
-    <link rel="alternate" hreflang="en" href="https://docs.godotengine.org/en/" />
-    <link rel="alternate" hreflang="de" href="https://docs.godotengine.org/de/" />
-    <link rel="alternate" hreflang="es" href="https://docs.godotengine.org/es/" />
-    <link rel="alternate" hreflang="fr" href="https://docs.godotengine.org/fr/" />
-    <link rel="alternate" hreflang="ko" href="https://docs.godotengine.org/ko/" />
-    <link rel="alternate" hreflang="pl" href="https://docs.godotengine.org/pl/" />
-    <link rel="alternate" hreflang="pt-br" href="https://docs.godotengine.org/pt-br/" />
-    <link rel="alternate" hreflang="uk" href="https://docs.godotengine.org/uk/" />
-    <link rel="alternate" hreflang="zh-cn" href="https://docs.godotengine.org/zh-cn/" />
-    <link rel="alternate" hreflang="x-default" href="https://docs.godotengine.org/" />
-    {{ super() }}
-{% endblock %}
+{% extends "!layout.html" -%}
+{% block linktags -%}
+  {% if godot_inject_language_links -%}
+  {% for alternate_lang in godot_docs_supported_languages -%}
+  {# Convert to ISO 639-1 format, e.g. zh_CN -> zh-cn -#}
+  {% set alternate_lang_href = alternate_lang.lower().replace("_", "-") -%}
+  <link rel="alternate" hreflang="{{ alternate_lang_href }}" href="{{ godot_docs_basepath }}{{ alternate_lang }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+  {% endfor -%}
+  <link rel="alternate" hreflang="x-default" href="{{ godot_docs_basepath }}{{ godot_default_lang }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+
+  <link rel="canonical" href="{{ godot_docs_basepath }}{{ lang_attr }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+  {% endif -%}
+  {{ super() }}
+{% endblock -%}

community/contributing/documentation_guidelines.rst

@@ -108,8 +108,12 @@ You can help to translate the official Godot documentation on our `Hosted Weblat
     :alt: Translation state
     :align: center
     :target: https://hosted.weblate.org/engage/godot-engine/?utm_source=widget
+    :width: 287
+    :height: 66
 
-There also is the official `Godot I18N repository <https://github.com/godotengine/godot-docs-l10n>`_. where you can see when the data was last synced.
+There also is the official
+`Godot i18n repository <https://github.com/godotengine/godot-docs-l10n>`_
+where you can see when the data was last synchronized.
 
 License
 -------

conf.py

@@ -46,8 +46,33 @@ if env_tags != None:
         tags.add(tag.strip())
 
 # Language / i18n
+
+supported_languages = {
+    "en": "Godot Engine (%s) documentation in English",
+    "de": "Godot Engine (%s) Dokumentation auf Deutsch",
+    "es": "Documentación de Godot Engine (%s) en español",
+    "fr": "Documentation de Godot Engine (%s) en français",
+    "fi": "Godot Engine (%s) dokumentaatio suomeksi",
+    "it": "Godot Engine (%s) documentazione in italiano",
+    "ja": "Godot Engine (%s)の日本語のドキュメント",
+    "ko": "Godot Engine (%s) 문서 (한국어)",
+    "pl": "Dokumentacja Godot Engine (%s) w języku polskim",
+    "pt_BR": "Documentação da Godot Engine (%s) em Português Brasileiro",
+    "ru": "Документация Godot Engine (%s) на русском языке",
+    "uk": "Документація до Godot Engine (%s) українською мовою",
+    "zh_CN": "Godot Engine (%s) 简体中文文档",
+}
+
 language = os.getenv("READTHEDOCS_LANGUAGE", "en")
-is_i18n = tags.has("i18n")
+if not language in supported_languages.keys():
+    print("Unknown language: " + language)
+    print("Supported languages: " + ", ".join(supported_languages.keys()))
+    print(
+        "The configured language is either wrong, or it should be added to supported_languages in conf.py. Falling back to 'en'."
+    )
+    language = "en"
+
+is_i18n = tags.has("i18n")  # noqa: F821
 
 exclude_patterns = ["_build"]
 
@@ -88,6 +113,8 @@ html_theme_options = {
     # 'navigation_depth': 4,  # Depth of the headers shown in the navigation bar
 }
 
+html_title = supported_languages[language] % version
+
 # VCS options: https://docs.readthedocs.io/en/latest/vcs.html#github
 html_context = {
     "display_github": not is_i18n,  # Integrate GitHub
@@ -95,6 +122,12 @@ html_context = {
     "github_repo": "godot-docs",  # Repo name
     "github_version": "master",  # Version
     "conf_py_path": "/",  # Path in the checkout to the docs root
+    "godot_inject_language_links": True,
+    "godot_docs_supported_languages": list(supported_languages.keys()),
+    "godot_docs_basepath": "https://docs.godotengine.org/",
+    "godot_docs_suffix": ".html",
+    "godot_default_lang": "en",
+    "godot_canonical_version": "stable",
 }
 
 html_logo = "img/docs_logo.png"
@@ -142,6 +175,8 @@ rst_epilog = """
 .. |weblate_widget| image:: https://hosted.weblate.org/widgets/godot-engine/{image_locale}/godot-docs/287x66-white.png
     :alt: Translation status
     :target: https://hosted.weblate.org/engage/godot-engine{target_locale}/?utm_source=widget
+    :width: 287
+    :height: 66
 """.format(
     image_locale="-" if language == "en" else language,
     target_locale="" if language == "en" else "/" + language,

robots.txt

@@ -1,7 +1,3 @@
 user-agent: *
-disallow: /*/3.2
-disallow: /*/3.1
-disallow: /*/3.0
-disallow: /*/2.1
 
 sitemap: https://docs.godotengine.org/sitemap.xml