Update antlr to 4.13 for 33% speed up on some inputs (#89)

* attempt at deploy script to work from 3p-packages-source

* Attempt at fixing package making scripts. In face of "testapp.py", line 121 NameEffor because of `fg.`

* Trying to understand why the 3p package still fails using explicit import names.

* Radical attempt

* Changes to the source to adapt to antlr 4.13

* suggested for test purpose, of course we should simply update the o3de fork and tags.

* this jar file is taken from the maven repository, https://repo1.maven.org/maven2/org/antlr/antlr4/4.13.2/

* regeneration result.

* update of the regen script for new antlr version

* revert antlr4 repository in the cmakelist to o3de fork now that the tag is ready.

---------

Signed-off-by: Vivien Oddou <[email protected]>

src/AzslcException.h

@@ -155,7 +155,7 @@ namespace AZ::ShaderCompiler
             BakeErrorMessage();
         }
 
-        const char* what() const NOEXCEPT
+        const char* what() const noexcept
         {
             return m_errorMessage.c_str();
         }

src/AzslcMain.cpp

@@ -23,7 +23,7 @@ namespace StdFs = std::filesystem;
 // For large features or milestones. Minor version allows for breaking changes. Existing tests can change.
 #define AZSLC_MINOR "8"   // last change: introduction of class inheritance
 // For small features or bug fixes. They cannot introduce breaking changes. Existing tests shouldn't change.
-#define AZSLC_REVISION "19"  // last change: fix crash in option ranks analysis in case of bodiless functions
+#define AZSLC_REVISION "20"  // last change: update antlrv4 to 4.13.2
 
 namespace AZ::ShaderCompiler
 {
@@ -43,11 +43,11 @@ namespace AZ::ShaderCompiler
         size_t maxToken = vocabulary.getMaxTokenType();
         for (size_t ii = 0; ii < maxToken; ++ii)
         {
-            string token = vocabulary.getLiteralName(ii);
+            string_view token = vocabulary.getLiteralName(ii);
             token = Trim(token, "\"'"); // because AntlR gives us e.g "'float'"
             if (!token.empty())         // empty when there is a complex rule (not a straight unconditional keyword)
             {
-                TypeClass tc = AnalyzeTypeClass(TentativeName{token});
+                TypeClass tc = AnalyzeTypeClass(TentativeName{string{token}});
                 bool accept = true;
                 if constexpr (!is_same_v<std::nullptr_t, std::remove_reference_t<decltype(tcFilter)>>)
                 {
@@ -56,7 +56,7 @@ namespace AZ::ShaderCompiler
 
                 if (tc == TypeClass::IsNotType)
                 {
-                    notTypes1.insert(token);
+                    notTypes1.insert(string{token});
                 }
 
                 if (accept)
@@ -220,30 +220,29 @@ namespace AZ::ShaderCompiler::Main
     /// This function will support the --ast option. It uses an AntlR facility and prettifies it.
     void PrintAst(tree::ParseTree* tree, azslParser& parser)
     {
-        // not sure why wstring, but I'm going along the AntlR's doc example.
-        std::wstring s = antlrcpp::s2ws(tree->toStringTree(&parser));
+        std::string s = tree->toStringTree(&parser);
         // hopefully easy to read indentator
-        std::wstring curindent = L"";
-        for (wchar_t c : s)
+        std::string curindent = "";
+        for (char c : s)
         {
-            if (c == L'(')
+            if (c == '(')
             {
-                std::wcout << "\n";
-                curindent += L"  ";
-                std::wcout << curindent << c;
+                std::cout << "\n";
+                curindent += "  ";
+                std::cout << curindent << c;
             }
-            else if (c == L')')
+            else if (c == ')')
             {
-                std::wcout << c << "\n";
+                std::cout << c << "\n";
                 curindent = curindent.substr(0, std::max(2_sz, curindent.size()) - 2);
-                std::wcout << curindent;
+                std::cout << curindent;
             }
             else
             {
-                std::wcout << c;
+                std::cout << c;
             }
         }
-        std::wcout << std::endl; // flush
+        std::cout << std::endl; // flush
     }
 
     /// this function supports the --visitsym option

src/AzslcSemanticOrchestrator.cpp

@@ -1670,7 +1670,7 @@ namespace AZ::ShaderCompiler
     optional<int64_t> SemanticOrchestrator::TryFoldSRGSemantic(azslParser::SrgSemanticContext* ctx, size_t semanticTokenType, bool required)
     {
         // const ref used, to extend the returned object's temporary life
-        const string& intrinsicVarNameFromLexer = m_lexer->getVocabulary().getLiteralName(semanticTokenType);
+        string_view intrinsicVarNameFromLexer = m_lexer->getVocabulary().getLiteralName(semanticTokenType);
         string_view intrinsicVarName = Trim(intrinsicVarNameFromLexer, "\'");
 
         auto semanticSymbol = LookupSymbol(UnqualifiedNameView{ intrinsicVarName });

src/CMakeLists.txt

@@ -13,8 +13,8 @@ FetchContent_Declare(
 # Antlr4 external dependency
 FetchContent_Declare(
   antlr4
-  GIT_REPOSITORY https://github.com/o3de/antlr4.git
-  GIT_TAG o3de-4.9.3
+  GIT_REPOSITORY https://github.com/o3de/antlr4/
+  GIT_TAG o3de-4.13.2
   SOURCE_SUBDIR runtime/Cpp
   CMAKE_ARGS "-DCMAKE_BUILD_TYPE=Release"
   GIT_SHALLOW TRUE

src/generated/azslLexer.h

@@ -1,5 +1,5 @@
 
-// Generated from ../azslLexer.g4 by ANTLR 4.9.3
+// Generated from ../azslLexer.g4 by ANTLR 4.13.2
 
 #pragma once
 
@@ -108,41 +108,34 @@ public:
   };
 
   explicit azslLexer(antlr4::CharStream *input);
-  ~azslLexer();
 
-  virtual std::string getGrammarFileName() const override;
-  virtual const std::vector<std::string>& getRuleNames() const override;
+  ~azslLexer() override;
 
-  virtual const std::vector<std::string>& getChannelNames() const override;
-  virtual const std::vector<std::string>& getModeNames() const override;
-  virtual const std::vector<std::string>& getTokenNames() const override; // deprecated, use vocabulary instead
-  virtual antlr4::dfa::Vocabulary& getVocabulary() const override;
 
-  virtual const std::vector<uint16_t> getSerializedATN() const override;
-  virtual const antlr4::atn::ATN& getATN() const override;
+  std::string getGrammarFileName() const override;
 
-private:
-  static std::vector<antlr4::dfa::DFA> _decisionToDFA;
-  static antlr4::atn::PredictionContextCache _sharedContextCache;
-  static std::vector<std::string> _ruleNames;
-  static std::vector<std::string> _tokenNames;
-  static std::vector<std::string> _channelNames;
-  static std::vector<std::string> _modeNames;
+  const std::vector<std::string>& getRuleNames() const override;
+
+  const std::vector<std::string>& getChannelNames() const override;
+
+  const std::vector<std::string>& getModeNames() const override;
+
+  const antlr4::dfa::Vocabulary& getVocabulary() const override;
 
-  static std::vector<std::string> _literalNames;
-  static std::vector<std::string> _symbolicNames;
-  static antlr4::dfa::Vocabulary _vocabulary;
-  static antlr4::atn::ATN _atn;
-  static std::vector<uint16_t> _serializedATN;
+  antlr4::atn::SerializedATNView getSerializedATN() const override;
 
+  const antlr4::atn::ATN& getATN() const override;
+
+  // By default the static state used to implement the lexer is lazily initialized during the first
+  // call to the constructor. You can call this function if you wish to initialize the static state
+  // ahead of time.
+  static void initialize();
+
+private:
 
   // Individual action functions triggered by action() above.
 
   // Individual semantic predicate functions triggered by sempred() above.
 
-  struct Initializer {
-    Initializer();
-  };
-  static Initializer _init;
 };
 

src/generated/azslParser.h

@@ -1,5 +1,5 @@
 
-// Generated from ../azslParser.g4 by ANTLR 4.9.3
+// Generated from ../azslParser.g4 by ANTLR 4.13.2
 
 #pragma once
 
@@ -156,13 +156,20 @@ public:
   };
 
   explicit azslParser(antlr4::TokenStream *input);
-  ~azslParser();
 
-  virtual std::string getGrammarFileName() const override;
-  virtual const antlr4::atn::ATN& getATN() const override { return _atn; };
-  virtual const std::vector<std::string>& getTokenNames() const override { return _tokenNames; }; // deprecated: use vocabulary instead.
-  virtual const std::vector<std::string>& getRuleNames() const override;
-  virtual antlr4::dfa::Vocabulary& getVocabulary() const override;
+  azslParser(antlr4::TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options);
+
+  ~azslParser() override;
+
+  std::string getGrammarFileName() const override;
+
+  const antlr4::atn::ATN& getATN() const override;
+
+  const std::vector<std::string>& getRuleNames() const override;
+
+  const antlr4::dfa::Vocabulary& getVocabulary() const override;
+
+  antlr4::atn::SerializedATNView getSerializedATN() const override;
 
 
   class CompilationUnitContext;
@@ -3218,26 +3225,16 @@ public:
   BorderColorEnumContext* borderColorEnum();
 
 
-  virtual bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override;
+  bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override;
+
   bool expressionSempred(ExpressionContext *_localctx, size_t predicateIndex);
   bool expressionExtSempred(ExpressionExtContext *_localctx, size_t predicateIndex);
 
-private:
-  static std::vector<antlr4::dfa::DFA> _decisionToDFA;
-  static antlr4::atn::PredictionContextCache _sharedContextCache;
-  static std::vector<std::string> _ruleNames;
-  static std::vector<std::string> _tokenNames;
-
-  static std::vector<std::string> _literalNames;
-  static std::vector<std::string> _symbolicNames;
-  static antlr4::dfa::Vocabulary _vocabulary;
-  static antlr4::atn::ATN _atn;
-  static std::vector<uint16_t> _serializedATN;
+  // By default the static state used to implement the parser is lazily initialized during the first
+  // call to the constructor. You can call this function if you wish to initialize the static state
+  // ahead of time.
+  static void initialize();
 
-
-  struct Initializer {
-    Initializer();
-  };
-  static Initializer _init;
+private:
 };
 

src/generated/azslParserBaseListener.cpp

@@ -1,5 +1,5 @@
 
-// Generated from ../azslParser.g4 by ANTLR 4.9.3
+// Generated from ../azslParser.g4 by ANTLR 4.13.2
 
 
 #include "azslParserBaseListener.h"

src/generated/azslParserBaseListener.h

@@ -1,5 +1,5 @@
 
-// Generated from ../azslParser.g4 by ANTLR 4.9.3
+// Generated from ../azslParser.g4 by ANTLR 4.13.2
 
 #pragma once
 

src/generated/azslParserListener.cpp

@@ -1,5 +1,5 @@
 
-// Generated from ../azslParser.g4 by ANTLR 4.9.3
+// Generated from ../azslParser.g4 by ANTLR 4.13.2
 
 
 #include "azslParserListener.h"

src/generated/azslParserListener.h

@@ -1,5 +1,5 @@
 
-// Generated from ../azslParser.g4 by ANTLR 4.9.3
+// Generated from ../azslParser.g4 by ANTLR 4.13.2
 
 #pragma once
 

src/regenerate_azsl_antlr.bat

@@ -2,16 +2,16 @@
 setlocal
 cd %~dp0
 cd generated
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -o java
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -o java
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -Dlanguage=Cpp
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -Dlanguage=Cpp
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -o java
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -o java
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -Dlanguage=Cpp
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -Dlanguage=Cpp
 @REM for some reason it's necessary to call two times, or new tokens are unrecognized
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -o java
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -o java
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -Dlanguage=Cpp
-java -cp "../external/antlr-4.9.3-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -Dlanguage=Cpp
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -o java
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -o java
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslLexer.g4 -Dlanguage=Cpp
+java -cp "../external/antlr4-4.13.2-complete.jar" org.antlr.v4.Tool ../azslParser.g4 -Dlanguage=Cpp
 
 cd java
 @REM for grun
-javac -cp "../../external/antlr-4.9.3-complete.jar" azsl*.java
+javac -cp "../../external/antlr4-4.13.2-complete.jar" azsl*.java