Simplify antlr4 runtime integration

- Adds o3de/antlr4 as a generation time dependency
- Removes unneeded python script to fetch git repos
- Adds antlr4_static as a target dependency
- Streamlines cmake options needed to ensure MSVC link compatibility

Signed-off-by: Jeremy Ong <[email protected]>

.gitignore

@@ -8,7 +8,6 @@ src/generated/*.tokens
 src/generated/java/
 .vs
 build
-src/external/antlr4*
 /src/.antlr/azslLexer.interp
 /src/.antlr/azslLexer.java
 /src/.antlr/azslLexer.tokens

prepare_solution_darwin.sh

@@ -12,10 +12,6 @@
 mkdir build
 mkdir build/release
 mkdir build/debug
-mkdir build/release/external
-mkdir build/release/external/antlr-runtime-cpp
-mkdir build/debug/external
-mkdir build/debug/external/antlr-runtime-cpp
 
 CMAKE='cmake'
 if ! command -v $CMAKE &> /dev/null
@@ -25,14 +21,6 @@ then
     echo "$OLDCMAKE not found in PATH. Defaulting to: $CMAKE"
 fi
 
-echo "Pulling ANTLR from git..."
-python3 pull_from_git.py --git-url https://github.com/o3de/antlr4.git --destination-dir src/external --git-tag o3de-4.7.1
-
-$CMAKE -DMAKE_BUILD_TYPE=Release -S "src/external/antlr4/runtime/Cpp/" -B "build/release/external/antlr4/runtime/Cpp/"
-pushd build/release/external/antlr4/runtime/Cpp
-make -j16
-popd
-
 $CMAKE -DMAKE_BUILD_TYPE=Release -S "src/" -B "build/release"
 pushd build/release
 echo "Building release..."

prepare_solution_linux.sh

@@ -12,13 +12,6 @@
 mkdir build
 mkdir build/release
 mkdir build/debug
-mkdir build/release/external
-mkdir build/release/external/antlr-runtime-cpp
-mkdir build/debug/external
-mkdir build/debug/external/antlr-runtime-cpp
-
-echo "Pulling ANTLR from git..."
-python3 pull_from_git.py --git-url https://github.com/o3de/antlr4.git --destination-dir src/external --git-tag o3de-4.7.1
 
 CMAKE='cmake'
 
@@ -36,11 +29,6 @@ echo "Release version:"
 ./azslc --version
 popd
 
-$CMAKE -DMAKE_BUILD_TYPE=Debug -S "src/external/antlr4/runtime/Cpp/" -B "build/debug/external/antlr4/runtime/Cpp/"
-pushd build/debug/external/antlr4/runtime/Cpp
-make -j16
-popd
-
 $CMAKE -DMAKE_BUILD_TYPE=Debug -S "src/" -B "build/debug"
 pushd build/debug
 echo "Building debug..."

prepare_solution_win.bat

@@ -15,29 +15,9 @@ set src_path=%curr_dir%\src
 set build_path=%curr_dir%\build\%platform%_%architecture%
 set cmake_exe=cmake
 
-REM ----------------------------------
-REM Pulling ANTLR from git
-python pull_from_git.py --git-url https://github.com/o3de/antlr4.git --destination-dir src/external --git-tag o3de-4.7.1 || goto :error
-
 if not exist "%build_path%" mkdir "%build_path%"
 cd "%build_path%" || goto :error
 
-REM ----------------------------------
-REM Generate ANTLR runtime project
-if not exist external mkdir external
-cd external
-if not exist antlr4 mkdir antlr4
-cd antlr4
-if not exist runtime mkdir runtime
-cd runtime
-if not exist Cpp mkdir Cpp
-cd Cpp
-
-%cmake_exe% -G %generator% "%src_path%\external\antlr4\runtime\Cpp\" || goto :error
-
-python -c "import sys; sys.stdout.write( open(sys.argv[1], 'r', encoding='utf-8_sig').read().replace('</RuntimeLibrary>', 'DLL</RuntimeLibrary>'))" runtime\antlr4_static.vcxproj > runtime\antlr4_static2.vcxproj || goto :error
-copy /Y "runtime\antlr4_static2.vcxproj" "runtime\antlr4_static.vcxproj"
-
 REM ----------------------------------
 REM Generate AZSLc project
 cd "%build_path%"

pull_from_git.py

@@ -1,175 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-# 
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-
-import argparse
-import sys
-import os
-import subprocess
-import platform
-from urllib.parse import urlparse
-
-def SwitchRootDirectory(destinationDir):
-    """
-    @returns a tuple previousDir, currentDir if successful
-    """
-    currentDir = os.getcwd()
-    destDir = currentDir
-    print(f"Current dir is {currentDir}")
-    if destinationDir is not None:
-        if not os.path.isabs(destinationDir):
-            destDir = os.path.join(currentDir, destinationDir)
-        else:
-            destDir = destinationDir
-        if not os.path.isdir(destDir):
-            raise Exception(f"{destDir} is NOT a directory")
-    # Change working directory before cloning the project
-    if currentDir != destDir:
-        os.chdir(destDir)
-    return currentDir, destDir
-
-def SubpArgs(args):
-    """
-    According to subcommand, when using shell=True, its recommended not to pass in an argument list but the full command line as a single string.
-    That means in the argument list in the configuration make sure to provide the proper escapements or double-quotes for paths with spaces
-
-    :param args: The list of arguments to transform
-    """
-    argString = " ".join([arg for arg in args])
-    print(f"Command: {argString}")
-    return argString
-
-def GetFilenameFromUrl(gitUrl):
-    aPath = urlparse(gitUrl)
-    filenameWithExtension = os.path.basename(aPath.path)
-    return os.path.splitext(filenameWithExtension)[0]
-
-def DeleteFolder(folder):
-    """
-    Use the system's remove folder command instead of os.rmdir
-    """
-    if platform.system() == 'Windows':
-        callResult = subprocess.run(subp_args(['rmdir', '/Q', '/S', str(folder.name)]),
-                                     shell=True,
-                                     capture_output=True,
-                                     cwd=str(folder.parent.absolute()))
-    else:
-        callResult = subprocess.run(subp_args(['rm', '-rf', str(folder.name)]),
-                                     shell=True,
-                                     capture_output=True,
-                                     cwd=str(folder.parent.absolute()))
-    if callResult.returncode != 0:
-        raise Exception(f"Unable to delete folder {str(folder)}: {str(call_result.stderr)}")
-
-
-def IsGitProjectCheckedOut(gitUrl, projectName, currentParentDir):
-    """
-    @param gitUrl The github url of the project
-    @param projectName [optional] If defined, this will be the local name of the project root folder
-    @param currentParentDir The full path of the current working directory which is the parent folder of the git project.
-    """
-    if projectName is None:
-        projectName = GetFilenameFromUrl(gitUrl)
-    projectPath = os.path.join(currentParentDir, projectName)
-    if not os.path.exists(projectPath):
-        return False
-    # The directory exists. Need to validate it's a valid git project
-    gitStashCmd = ['git', 'stash']
-    callResult = subprocess.run(SubpArgs(gitStashCmd),
-                                 shell=True,
-                                 capture_output=True,
-                                 cwd=projectPath)
-    if callResult.returncode != 0:
-        # Not a valid git folder, okay to remove and re-clone
-        DeleteFolder(projectPath)
-        return False
-
-    # Do not re-pull for now.
-    ## Do a re-pull
-    #gitPullCmd = ['git', 'pull']
-    #callResult = subprocess.run(SubpArgs(gitPullCmd),
-    #                             shell=True,
-    #                             capture_output=True,
-    #                             cwd=projectPath)
-    #if callResult.returncode != 0:
-    #    raise Exception(f"Error pulling source from GitHub: {callResult.stderr.decode('UTF-8', 'ignore')}")
-    return True
-
-
-def FetchGitProject(gitUrl, projectName, destinationDir, gitTag, gitCommit):
-    """
-    @param gitUrl The github url of the project
-    @param projectName [optional] If defined, this will be the local name of the project root folder
-    @param destinationDir [optional] If defined, this is the directory where the project will be cloned into
-    """
-    currentDir, destinationDir = SwitchRootDirectory(destinationDir)
-    
-    # Before cloning see if the repo already exists.
-    if IsGitProjectCheckedOut(gitUrl, projectName, destinationDir):
-        print(f"Git project {gitUrl} already cloned under parent directory {destinationDir}")
-        return
-
-    cloneCmd = ['git',
-                'clone',
-                '--single-branch',
-                '--recursive']
-    if gitTag is not None:
-        cloneCmd.extend(['--branch', gitTag])
-    cloneCmd.append(gitUrl)
-    if projectName is not None:
-        cloneCmd.append(projectName)
-    else:
-        projectName = GetFilenameFromUrl(gitUrl)
-    cloneResult = subprocess.run(SubpArgs(cloneCmd),
-                                 shell=True,
-                                 capture_output=True,
-                                 cwd=destinationDir)
-    if cloneResult.returncode != 0:
-        raise Exception(f"Error cloning from GitHub: {cloneResult.stderr.decode('UTF-8', 'ignore')}")
-    if gitCommit is not None:
-        # Allow the package to specify a specific commit to check out. This is useful for upstream repos that do
-        # not tag their releases.
-        # Checking out 
-        checkoutResult = subprocess.run(
-            SubpArgs(['git', 'checkout', gitCommit]),
-            shell=True,
-            capture_output=True,
-            cwd=projectName)
-        if checkoutResult.returncode != 0:
-            raise Exception(f"Error checking out {gitCommit}: {checkoutResult.stderr.decode('UTF-8', 'ignore')}")
-    print(f"Successfully cloned Git project {gitUrl} under parent directory {destinationDir}")
-
-if __name__ == '__main__':
-    try:
-        argParser = argparse.ArgumentParser(
-            description="Tool to pull a github project if not already pulled",
-            formatter_class=argparse.RawDescriptionHelpFormatter)
-        argParser.add_argument('--git-url',
-                            help='The github url of the project',
-                            required=True)
-        argParser.add_argument('--name',
-                            help='If defined, this will be the local name of the project root folder',
-                            required=False)
-        argParser.add_argument('--destination-dir',
-                            help='If defined, this is the directory where the project will be cloned into',
-                            required=False)
-        argParser.add_argument('--git-tag',
-                            help='If defined, the project will be checked out at this tag.',
-                            required=False)
-        argParser.add_argument('--git-commit',
-                            help='If defined, the project will be checked out at this commit hash.',
-                            required=False)
-        parsedArgs = argParser.parse_args(sys.argv[1:])
-        FetchGitProject(
-            gitUrl = parsedArgs.git_url,
-            projectName = parsedArgs.name,
-            destinationDir = parsedArgs.destination_dir,
-            gitTag = parsedArgs.git_tag,
-            gitCommit = parsedArgs.git_commit)
-    except Exception as err:
-        print(err)
-        sys.exit(1)

src/AzslcException.h

@@ -286,6 +286,7 @@ namespace AZ::ShaderCompiler
     public:
         explicit AzslParserEventListener(PreprocessorLineDirectiveFinder& lineDirectiveFinder) : m_lineDirectiveFinder(lineDirectiveFinder) {}
         AzslParserEventListener() = delete;
+        ~AzslParserEventListener() override = default;
 
         void syntaxError(antlr4::Recognizer* recognizer, antlr4::Token* offendingSymbol, size_t line,
             size_t charPositionInLine, const string &msg, std::exception_ptr e) override

src/CMakeLists.txt

@@ -2,8 +2,6 @@ cmake_minimum_required(VERSION 3.21)
 
 project (Azslc)
 
-SET (CMAKE_CXX_STANDARD 17)
-
 include(FetchContent)
 
 FetchContent_Declare(
@@ -11,39 +9,17 @@ FetchContent_Declare(
   GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git
   GIT_TAG 3ce61a169297f5fe7d4b08bdbfa82290950d6f7f
 )
-FetchContent_MakeAvailable(cli11)
-
-if (MSVC)
-  add_compile_options("/std:c++17")
-  add_compile_options("/permissive-")
-  add_compile_options("/Zc:__cplusplus")
-  add_compile_options("/utf-8")
-  add_compile_options("/MP")
-else()
-  add_compile_options("--std=c++17")
-endif()
-
-  set(CompilerFlags
-        CMAKE_CXX_FLAGS
-        CMAKE_CXX_FLAGS_DEBUG
-        CMAKE_CXX_FLAGS_RELEASE
-        CMAKE_C_FLAGS
-        CMAKE_C_FLAGS_DEBUG
-        CMAKE_C_FLAGS_RELEASE
-        )
 
-if (MSVC)
-  foreach(CompilerFlag ${CompilerFlags})
-    string(REPLACE "/MT" "/MD" ${CompilerFlag} "${${CompilerFlag}}")
-  endforeach()
-else()
-  set (CMAKE_CXX_FLAGS "-Wno-logical-op-parentheses")
-endif()
+# Antlr4 external dependency
+FetchContent_Declare(
+  antlr4
+  GIT_REPOSITORY https://github.com/o3de/antlr4.git
+  GIT_TAG o3de-4.7.1
+  SOURCE_SUBDIR runtime/Cpp
+  CMAKE_ARGS "-DCMAKE_BUILD_TYPE=Release"
+)
 
-if (MSVC)
-  add_definitions(-D_CRT_SECURE_NO_WARNINGS)
-  add_definitions(-D_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
-endif()
+FetchContent_MakeAvailable(cli11 antlr4)
 
 # get all chars that are not separators before the end
 string(REGEX MATCH "[^/\\]*$" buildPlatform ${CMAKE_BINARY_DIR})
@@ -70,17 +46,12 @@ file(GLOB azslc_GeneratedSrc
   "${PROJECT_SOURCE_DIR}/generated/*.cpp"
 )
 
-if (MSVC)
-  include_external_msproject(antlr4_static ${CMAKE_BINARY_DIR}/external/antlr4/runtime/Cpp/runtime/antlr4_static.vcxproj)
-endif()
-
 set( ANTLR4CPP_INCLUDE_DIRS
-  ${PROJECT_SOURCE_DIR}/external/antlr4/runtime/Cpp/runtime/src
-  ${PROJECT_SOURCE_DIR}/external/antlr4/runtime/Cpp/runtime/src/dfa
-  ${PROJECT_SOURCE_DIR}/external/antlr4/runtime/Cpp/runtime/src/misc
-  ${PROJECT_SOURCE_DIR}/external/antlr4/runtime/Cpp/runtime/src/tree
+  ${antlr4_SOURCE_DIR}/runtime/Cpp/runtime/src
+  ${antlr4_SOURCE_DIR}/runtime/Cpp/runtime/src/dfa
+  ${antlr4_SOURCE_DIR}/runtime/Cpp/runtime/src/misc
+  ${antlr4_SOURCE_DIR}/runtime/Cpp/runtime/src/tree
 )
-# add antrl4cpp artifacts to project environment
 
 set( MPARK_VARIANT_INCLUDE_DIRS
   ${PROJECT_SOURCE_DIR}/external/mpark-variant/master
@@ -90,25 +61,19 @@ set( TINY_OPTIONAL_INCLUDE_DIRS
   ${PROJECT_SOURCE_DIR}/external/tiny
 )
 
-include_directories(
-    ${PROJECT_SOURCE_DIR}
-    ${PROJECT_SOURCE_DIR}/external
-    ${ANTLR4CPP_INCLUDE_DIRS}
-    ${MPARK_VARIANT_INCLUDE_DIRS}
-    ${TINY_OPTIONAL_INCLUDE_DIRS}
-)
+add_executable(azslc ${azslc_LocalSRC} ${azslc_PlatformSRC} ${azslc_GeneratedSrc} ${azslc_ExternalSRC})
 
-if (MSVC)
-  link_directories(
-    ${CMAKE_BINARY_DIR}/external/antlr4/runtime/Cpp/dist/${CMAKE_BUILD_TYPE}
-  )
-elseif (UNIX)
-  link_directories(
-    ${CMAKE_BINARY_DIR}/external/antlr4/runtime/Cpp/dist/
-  )
-endif()
+target_compile_features(azslc PRIVATE cxx_std_17)
 
-add_executable(azslc ${azslc_LocalSRC} ${azslc_PlatformSRC} ${azslc_GeneratedSrc} ${azslc_ExternalSRC})
+target_include_directories(
+  azslc
+  PRIVATE
+  ${PROJECT_SOURCE_DIR}
+  ${PROJECT_SOURCE_DIR}/external
+  ${ANTLR4CPP_INCLUDE_DIRS}
+  ${MPARK_VARIANT_INCLUDE_DIRS}
+  ${TINY_OPTIONAL_INCLUDE_DIRS}
+)
 
 source_group("azslc" FILES ${azslc_LocalSRC})
 source_group("generated" FILES ${azslc_GeneratedSrc})
@@ -116,18 +81,24 @@ source_group("external" FILES ${azslc_ExternalSRC})
 source_group("platforms" FILES ${azslc_PlatformSRC})
 
 if (MSVC)
-  target_link_libraries(azslc PRIVATE antlr4-runtime-static)
+  set_property(TARGET azslc PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+  target_compile_options(azslc
+    PRIVATE
+    "/MP" # Build with multiple processes
+    "/permissive-"
+    "/Zc:__cplusplus"
+    "/utf-8")
+  target_compile_definitions(azslc
+    PRIVATE
+    _CRT_SECURE_NO_WARNINGS
+    _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
 elseif (UNIX AND NOT APPLE)
-  target_link_libraries(azslc PRIVATE libantlr4-runtime.a stdc++fs)
-elseif (UNIX)
-  target_link_libraries(azslc PRIVATE libantlr4-runtime.a)
+  target_link_libraries(azslc PRIVATE stdc++fs)
 endif()
 
-target_link_libraries(azslc PRIVATE CLI11::CLI11)
+target_link_libraries(azslc PRIVATE CLI11::CLI11 antlr4_static)
 
 if (MSVC)
-  add_dependencies(azslc antlr4_static)
-
   add_custom_command(TARGET azslc POST_BUILD
       COMMAND ${CMAKE_COMMAND} -E make_directory "${PROJECT_SOURCE_DIR}/../bin/${buildPlatform}/$(Configuration)"
       COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/$(Configuration)/azslc.exe" "${PROJECT_SOURCE_DIR}/../bin/${buildPlatform}/$(Configuration)"