Update to pull_and_build_from_git.py (#17)

- Add a new field 'cmake_find_source' for the build_config.json to accept a straight source file that will not be processed
- Keep existing 'cmake_find_template'
- Update DirectXShaderCompiler to use new field since it doesnt need templating
- Minor fix to DirectXShaderCompiler scripts
- Removing redundant checks for cmake_find_template & cmake_find_source, replacing it with asserts
- Remove Debug builds from the windows DXC build process (release only)
- Fix package version to reflect package_build_list

Signed-off-by: Steve Pham <[email protected]>

Scripts/extras/pull_and_build_from_git.py

@@ -43,8 +43,11 @@ The following keys can exist at the root level or the target-platform level:
                           'prebuilt_args' below.
 * prebuild_args         : (required if  prebuilt_source is set) A map of target subfolders within the target 3rd party folder against a glob pattern of
                           file(s) to copy to the target subfolders.
-* cmake_find_template   : (required if prebuilt_source is not set) The name of the template file that is used to generate the find*.cmake file
+* cmake_find_source     : The name of the source find*.cmake file that will be used in the target package
                           that is ingested by the lumberyard 3P system.
+* cmake_find_template   : If the find*.cmake in the target package requires template processing, then this is name of the template file that is used to 
+                          generate the contents of the find*.cmake file in the target package. 
+                          * Note that either 'cmake_find_source' or 'cmake_fine_template' must be declared.
 * cmake_find_target     : (required if prebuilt_source is not set) The name of the target find*.cmake file that is generated based on the template file and 
                           additional arguments (described below)
 
@@ -52,7 +55,7 @@ The following keys can exist at the root level or the target-platform level:
                           to restrict building to a specific configuration rather than building all configurations 
                           (provided by the default value: ['Debug', 'Release'])
 * patch_file            : (optional) Option patch file to apply to the synced source before performing a build
-* source_path            : (optional) Option to provide a path to the project source rather than getting it from github
+* source_path           : (optional) Option to provide a path to the project source rather than getting it from github
 * git_skip              : (optional) Option to skip all git commands, requires source_path
 
 
@@ -171,11 +174,18 @@ class PackageInfo(object):
         self.package_version = _get_value("package_version")
         self.patch_file = _get_value("patch_file", required=False)
         self.git_commit = _get_value("git_commit", required=False)
-        self.cmake_find_template = _get_value("cmake_find_template")
+        self.cmake_find_template = _get_value("cmake_find_template", required=False)
+        self.cmake_find_source = _get_value("cmake_find_source", required=False)
         self.cmake_find_target = _get_value("cmake_find_target")
         self.cmake_find_template_custom_indent = _get_value("cmake_find_template_custom_indent", default=1)
         self.additional_src_files = _get_value("additional_src_files", required=False)
 
+        if self.cmake_find_template and self.cmake_find_source:
+            raise BuildError("Bad build config file. 'cmake_find_template' and 'cmake_find_source' cannot both be set in the configuration.")            
+        if not self.cmake_find_template and not self.cmake_find_source:
+            raise BuildError("Bad build config file. 'cmake_find_template' or 'cmake_find_source' must be set in the configuration.")
+
+
     def write_package_info(self, install_path):
         """
         Write to the target 'PackageInfo.json' file for the package
@@ -294,7 +304,9 @@ class BuildInfo(object):
     This is the Build management class that will perform the entire build from source and preparing a folder for packaging
     """
 
-    def __init__(self, package_info, platform_config, base_folder, build_folder, package_install_root, custom_toolchain_file, cmake_command, clean_build, cmake_find_template, prebuilt_source, prebuilt_args, src_folder, skip_git):
+    def __init__(self, package_info, platform_config, base_folder, build_folder, package_install_root, 
+                 custom_toolchain_file, cmake_command, clean_build, cmake_find_template, 
+                 cmake_find_source, prebuilt_source, prebuilt_args, src_folder, skip_git):
         """
         Initialize the Build management object with information needed
 
@@ -307,12 +319,16 @@ class BuildInfo(object):
         :param cmake_command:           The cmake executable command to use for cmake
         :param clean_build:             Option to clean any existing build folder before proceeding
         :param cmake_find_template:     The template for the find*.cmake generated file
+        :param cmake_find_source:       The source file for the find*.cmake generated file
         :param prebuilt_source:         If provided, the git fetch / build flow will be replaced with a copy from a prebuilt folder
         :param prebuilt_args:           If prebuilt_source is provided, then this argument is required to specify the copy rules to assemble the package from the prebuilt package
         :param src_folder:              Path to the source code / where to clone the git repo.
         :param skip_git:                If true skip all git interaction and .
         """
 
+        assert (cmake_find_template is not None and cmake_find_source is None) or \
+                (cmake_find_template is None and cmake_find_source is not None), "Either cmake_find_template or cmake_find_source must be set, but not both"
+
         self.package_info = package_info
         self.platform_config = platform_config
         self.custom_toolchain_file = custom_toolchain_file
@@ -325,11 +341,13 @@ class BuildInfo(object):
         self.build_install_folder = self.package_install_root / package_info.package_name
         self.clean_build = clean_build
         self.cmake_find_template = cmake_find_template
+        self.cmake_find_source = cmake_find_source
         self.build_configs = platform_config.get('build_configs', ['Debug', 'Release'])
         self.prebuilt_source = prebuilt_source
         self.prebuilt_args = prebuilt_args
         self.skip_git = skip_git
 
+
     def clone_to_local(self):
         """
         Perform a clone to the local temp folder
@@ -651,24 +669,31 @@ class BuildInfo(object):
         Generate the find*.cmake file for the library
         """
 
-        template_file_content = self.cmake_find_template.read_text("UTF-8", "ignore")
+        if self.cmake_find_template is not None:
 
-        def _build_list_str(indent, key):
-            list_items = self.platform_config.get(key, [])
-            indented_list_items = []
-            for list_item in list_items:
-                indented_list_items.append(f'{" "*(indent*4)}{list_item}')
-            return '\n'.join(indented_list_items)
+            template_file_content = self.cmake_find_template.read_text("UTF-8", "ignore")
 
-        cmake_find_template_def_ident_level = self.package_info.cmake_find_template_custom_indent
+            def _build_list_str(indent, key):
+                list_items = self.platform_config.get(key, [])
+                indented_list_items = []
+                for list_item in list_items:
+                    indented_list_items.append(f'{" "*(indent*4)}{list_item}')
+                return '\n'.join(indented_list_items)
 
-        template_env = {
-            "CUSTOM_ADDITIONAL_COMPILE_DEFINITIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_compile_definitions'),
-            "CUSTOM_ADDITIONAL_LINK_OPTIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_link_options'),
-            "CUSTOM_ADDITIONAL_LIBRARIES": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_libraries')
-        }
+            cmake_find_template_def_ident_level = self.package_info.cmake_find_template_custom_indent
+
+            template_env = {
+                "CUSTOM_ADDITIONAL_COMPILE_DEFINITIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_compile_definitions'),
+                "CUSTOM_ADDITIONAL_LINK_OPTIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_link_options'),
+                "CUSTOM_ADDITIONAL_LIBRARIES": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_libraries')
+            }
+
+            find_cmake_content = string.Template(template_file_content).substitute(template_env)
+
+        elif self.cmake_find_source is not None:
+
+            find_cmake_content = self.cmake_find_source.read_text("UTF-8", "ignore")
 
-        find_cmake_content = string.Template(template_file_content).substitute(template_env)
 
         target_cmake_find_script = self.package_install_root / self.package_info.cmake_find_target
         target_cmake_find_script.write_text(find_cmake_content)
@@ -803,15 +828,31 @@ def prepare_build(platform_name, base_folder, build_folder, package_root_folder,
                                target_platform_name=platform_name,
                                target_platform_config=target_platform_config)
 
-    # Validate the cmake find template
-    if not package_info.cmake_find_template:
-        raise BuildError("Missing 'cmake_find_template' entry in build config")
-    if os.path.isabs(package_info.cmake_find_template):
-        cmake_find_template_path = pathlib.Path(package_info.cmake_find_template)
-    else:
+    cmake_find_template_path = None
+    cmake_find_source_path = None
+
+    if package_info.cmake_find_template is not None:
+
+        # Validate the cmake find template
+        if os.path.isabs(package_info.cmake_find_template):
+            raise BuildError("Invalid 'cmake_find_template' entry in build config. Absolute paths are not allowed, must be relative to the package base folder.")
+        
         cmake_find_template_path = base_folder_path / package_info.cmake_find_template
-    if not cmake_find_template_path.is_file():
-        raise BuildError("Invalid 'cmake_find_template' entry in build config")
+        if not cmake_find_template_path.is_file():
+            raise BuildError("Invalid 'cmake_find_template' entry in build config")
+
+    elif package_info.cmake_find_source is not None:
+
+        # Validate the cmake find source
+        if os.path.isabs(package_info.cmake_find_source):
+            raise BuildError("Invalid 'cmake_find_source' entry in build config. Absolute paths are not allowed, must be relative to the package base folder.")
+
+        cmake_find_source_path = base_folder_path / package_info.cmake_find_source
+        if not cmake_find_source_path.is_file():
+            raise BuildError("Invalid 'cmake_find_source' entry in build config")
+
+    else:
+        raise BuildError("Bad build config file. 'cmake_find_template' or 'cmake_find_template' must be specified.")            
 
     return BuildInfo(package_info=package_info,
                      platform_config=target_platform_config,
@@ -822,6 +863,7 @@ def prepare_build(platform_name, base_folder, build_folder, package_root_folder,
                      cmake_command=cmake_command,
                      clean_build=clean,
                      cmake_find_template=cmake_find_template_path,
+                     cmake_find_source=cmake_find_source_path,
                      prebuilt_source=prebuilt_source,
                      prebuilt_args=prebuilt_args,
                      src_folder=src_folder_path,

package-system/DirectXShaderCompiler/FindDirectXShaderCompilerDxc.cmake.Linux

@@ -7,27 +7,27 @@
 #
 
 set(MY_NAME "DirectXShaderCompilerDxc")
-set(TARGET_WITH_NAMESPACE "3rdParty::$${MY_NAME}")
-if (TARGET $${TARGET_WITH_NAMESPACE})
+set(TARGET_WITH_NAMESPACE "3rdParty::${MY_NAME}")
+if (TARGET ${TARGET_WITH_NAMESPACE})
     return()
 endif()
 
 set(output_subfolder "Builders/DirectXShaderCompiler")
-set($${MY_NAME}_BINARY_DIR $${CMAKE_CURRENT_LIST_DIR}/$${MY_NAME}/bin)
-set($${MY_NAME}_LIB_DIR $${CMAKE_CURRENT_LIST_DIR}/$${MY_NAME}/lib)
+set(${MY_NAME}_BINARY_DIR ${CMAKE_CURRENT_LIST_DIR}/${MY_NAME}/bin)
+set(${MY_NAME}_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/${MY_NAME}/lib)
 
-add_library($${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
+add_library(${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
 
-set($${MY_NAME}_BIN_RUNTIME_DEPENDENCIES
-        $${$${MY_NAME}_BINARY_DIR}/dxc
-        $${$${MY_NAME}_BINARY_DIR}/dxc-3.7
+set(${MY_NAME}_BIN_RUNTIME_DEPENDENCIES
+        ${${MY_NAME}_BINARY_DIR}/dxc
+        ${${MY_NAME}_BINARY_DIR}/dxc-3.7
         )
-ly_add_target_files(TARGETS $${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "$${output_subfolder}/bin" FILES $${$${MY_NAME}_BIN_RUNTIME_DEPENDENCIES})
+ly_add_target_files(TARGETS ${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "${output_subfolder}/bin" FILES ${${MY_NAME}_BIN_RUNTIME_DEPENDENCIES})
 
-set($${MY_NAME}_LIB_RUNTIME_DEPENDENCIES
-        $${$${MY_NAME}_LIB_DIR}/libdxcompiler.so
-        $${$${MY_NAME}_LIB_DIR}/libdxcompiler.so.3.7
+set(${MY_NAME}_LIB_RUNTIME_DEPENDENCIES
+        ${${MY_NAME}_LIB_DIR}/libdxcompiler.so
+        ${${MY_NAME}_LIB_DIR}/libdxcompiler.so.3.7
         )
-ly_add_target_files(TARGETS $${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "$${output_subfolder}/lib" FILES $${$${MY_NAME}_LIB_RUNTIME_DEPENDENCIES})
+ly_add_target_files(TARGETS ${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "${output_subfolder}/lib" FILES ${${MY_NAME}_LIB_RUNTIME_DEPENDENCIES})
 
-set($${MY_NAME}_FOUND True)
+set(${MY_NAME}_FOUND True)

package-system/DirectXShaderCompiler/FindDirectXShaderCompilerDxc.cmake.Mac

@@ -6,27 +6,27 @@
 #
 #
 set(MY_NAME "DirectXShaderCompilerDxc")
-set(TARGET_WITH_NAMESPACE "3rdParty::$${MY_NAME}")
-if (TARGET $${TARGET_WITH_NAMESPACE})
+set(TARGET_WITH_NAMESPACE "3rdParty::${MY_NAME}")
+if (TARGET ${TARGET_WITH_NAMESPACE})
     return()
 endif()
 
 set(output_subfolder "Builders/DirectXShaderCompiler")
-set($${MY_NAME}_BINARY_DIR $${CMAKE_CURRENT_LIST_DIR}/$${MY_NAME}/bin)
-set($${MY_NAME}_LIB_DIR $${CMAKE_CURRENT_LIST_DIR}/$${MY_NAME}/lib)
+set(${MY_NAME}_BINARY_DIR ${CMAKE_CURRENT_LIST_DIR}/${MY_NAME}/bin)
+set(${MY_NAME}_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/${MY_NAME}/lib)
 	
-add_library($${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
+add_library(${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
 
-set($${MY_NAME}_BIN_RUNTIME_DEPENDENCIES
-        $${$${MY_NAME}_BINARY_DIR}/dxc
-        $${$${MY_NAME}_BINARY_DIR}/dxc-3.7
+set(${MY_NAME}_BIN_RUNTIME_DEPENDENCIES
+        ${${MY_NAME}_BINARY_DIR}/dxc
+        ${${MY_NAME}_BINARY_DIR}/dxc-3.7
         )
-ly_add_target_files(TARGETS $${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "$${output_subfolder}/bin" FILES $${$${MY_NAME}_BIN_RUNTIME_DEPENDENCIES})
+ly_add_target_files(TARGETS ${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "${output_subfolder}/bin" FILES ${${MY_NAME}_BIN_RUNTIME_DEPENDENCIES})
 
-set($${MY_NAME}_LIB_RUNTIME_DEPENDENCIES
-        $${$${MY_NAME}_LIB_DIR}/libdxcompiler.dylib
-        $${$${MY_NAME}_LIB_DIR}/libdxcompiler.3.7.dylib
+set(${MY_NAME}_LIB_RUNTIME_DEPENDENCIES
+        ${${MY_NAME}_LIB_DIR}/libdxcompiler.dylib
+        ${${MY_NAME}_LIB_DIR}/libdxcompiler.3.7.dylib
         )
-ly_add_target_files(TARGETS $${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "$${output_subfolder}/lib" FILES $${$${MY_NAME}_LIB_RUNTIME_DEPENDENCIES})
+ly_add_target_files(TARGETS ${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY "${output_subfolder}/lib" FILES ${${MY_NAME}_LIB_RUNTIME_DEPENDENCIES})
 
-set($${MY_NAME}_FOUND True)
+set(${MY_NAME}_FOUND True)

package-system/DirectXShaderCompiler/FindDirectXShaderCompilerDxc.cmake.Windows

@@ -7,21 +7,21 @@
 #
 
 set(MY_NAME "DirectXShaderCompilerDxc")
-set(TARGET_WITH_NAMESPACE "3rdParty::$${MY_NAME}")
-if (TARGET $${TARGET_WITH_NAMESPACE})
+set(TARGET_WITH_NAMESPACE "3rdParty::${MY_NAME}")
+if (TARGET ${TARGET_WITH_NAMESPACE})
     return()
 endif()
 
 set(output_subfolder "Builders/DirectXShaderCompiler")
-set($${MY_NAME}_BINARY_DIR $${CMAKE_CURRENT_LIST_DIR}/$${MY_NAME}/bin)
+set(${MY_NAME}_BINARY_DIR ${CMAKE_CURRENT_LIST_DIR}/${MY_NAME}/bin)
 
-add_library($${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
+add_library(${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
 
-set($${MY_NAME}_RUNTIME_DEPENDENCIES
-		$${$${MY_NAME}_BINARY_DIR}/Release/dxc.exe
-		$${$${MY_NAME}_BINARY_DIR}/Release/dxil.dll
-		$${$${MY_NAME}_BINARY_DIR}/Release/dxcompiler.dll)
-ly_add_target_files(TARGETS $${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY $${output_subfolder} FILES $${$${MY_NAME}_RUNTIME_DEPENDENCIES})
+set(${MY_NAME}_RUNTIME_DEPENDENCIES
+		${${MY_NAME}_BINARY_DIR}/Release/dxc.exe
+		${${MY_NAME}_BINARY_DIR}/Release/dxil.dll
+		${${MY_NAME}_BINARY_DIR}/Release/dxcompiler.dll)
+ly_add_target_files(TARGETS ${TARGET_WITH_NAMESPACE} OUTPUT_SUBDIRECTORY ${output_subfolder} FILES ${${MY_NAME}_RUNTIME_DEPENDENCIES})
 
 
-set($${MY_NAME}_FOUND True)
+set(${MY_NAME}_FOUND True)

package-system/DirectXShaderCompiler/build_config.json

@@ -10,7 +10,7 @@
    "Platforms":{
       "Windows":{
          "Windows":{
-            "cmake_find_template":"FindDirectXShaderCompilerDxc.cmake.Windows",
+            "cmake_find_source":"FindDirectXShaderCompilerDxc.cmake.Windows",
             "custom_build_cmd": [
                "build_dxc_windows.cmd"
             ],
@@ -22,7 +22,7 @@
       "Darwin":{
          "Mac":{
             "build_configs": ["Release"],
-            "cmake_find_template":"FindDirectXShaderCompilerDxc.cmake.Mac",
+            "cmake_find_source":"FindDirectXShaderCompilerDxc.cmake.Mac",
             "cmake_generate_args":[
                "-DCMAKE_BUILD_TYPE=Release",
                "-DCMAKE_INSTALL_LIBDIR=\"lib/Release\"",
@@ -62,7 +62,7 @@
       "Linux":{
          "Linux":{
             "build_configs": ["Release"],
-            "cmake_find_template":"FindDirectXShaderCompilerDxc.cmake.Linux",
+            "cmake_find_source":"FindDirectXShaderCompilerDxc.cmake.Linux",
             "cmake_generate_args":[
                "-GNinja",
                "-DCMAKE_BUILD_TYPE=Release",