Jelajahi Sumber

Refactored the project external subdirectory visit workflow (#8388)

* Refactored the project external subdirectory visit workflow

The order in which directories are visited when configuring a project is
now as follows

1. Engine Code directories
1. User supplied LY_EXTERNAL_SUBDIRS via the CMake Cache Variable.
1. Engine Gem directories("external_subdirectories")
1. Engine references to registered gems("gem_names")
1. Project references to registered gems("gem_names")
1. The Project root directory
1. The Project Gem directories("external_subdirectories")

This change allows projects to use cmake functionality exposed by gem's
CMake files.

resolves #8376

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Removed circular dependencies from the Atom set of gems gem.json files.

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Added support to the External Subdirectory visit logic to reorder gems

Any gems that are listed in the "Dependencies" section of the gem.json
file are now visited before that gem.

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Remove unnecessary warning in launcher_generator.cmake

The "project_name" field in the project.json has already been read in
Projects.cmake

Signed-off-by: lumberyard-employee-dm <[email protected]>
lumberyard-employee-dm 3 tahun lalu
induk
melakukan
456fbe1dee

+ 7 - 8
CMakeLists.txt

@@ -53,18 +53,13 @@ add_o3de_manifest_json_external_subdirectories()
 # Add the projects first so the Launcher can find them
 include(cmake/Projects.cmake)
 
-if(LY_EXTERNAL_SUBDIRS)
-    set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${LY_EXTERNAL_SUBDIRS})
-endif()
 
 if(NOT INSTALLED_ENGINE)
-    # Add external subdirectories listed in the engine.json. LY_EXTERNAL_SUBDIRS is a cache variable so the user can add extra
-    # external subdirectories. This should go before adding the rest of the targets so the targets are available to the launcher.
+    # Add external subdirectories listed in the engine.json.
+    # LY_EXTERNAL_SUBDIRS is a cache variable so the user can add extra
+    # external subdirectories.
     add_engine_json_external_subdirectories()
 
-    # Invoke add_subdirectory on external subdirectories that should be used a this point
-    add_subdirectory_on_external_subdirs()
-
     # Add the rest of the targets
     add_subdirectory(Assets)
     add_subdirectory(Code)
@@ -74,6 +69,10 @@ if(NOT INSTALLED_ENGINE)
     add_subdirectory(Templates)
     add_subdirectory(Tools)
 
+    # Invoke add_subdirectory on all external subdirectories
+    # that is in use by the union of projects specified by path in LY_PROJECTS
+    add_subdirectory_on_external_subdirs()
+
 else()
     ly_find_o3de_packages()
     add_subdirectory_on_external_subdirs()

+ 45 - 58
Code/LauncherUnified/launcher_generator.cmake

@@ -11,19 +11,13 @@ set_property(GLOBAL PROPERTY LAUNCHER_UNIFIED_BINARY_DIR ${CMAKE_CURRENT_BINARY_
 # Launcher targets for a project need to be generated when configuring a project.
 # When building the engine source, this file will be included by LauncherUnified's CMakeLists.txt
 # When using an installed engine, this file will be included by the FindLauncherGenerator.cmake script
-get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
-foreach(project_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJECTS)
+get_property(O3DE_PROJECTS_NAME GLOBAL PROPERTY O3DE_PROJECTS_NAME)
+foreach(project_name project_path IN ZIP_LISTS O3DE_PROJECTS_NAME LY_PROJECTS)
     
     # Computes the realpath to the project
     # If the project_path is relative, it is evaluated relative to the ${LY_ROOT_FOLDER}
     # Otherwise the the absolute project_path is returned with symlinks resolved
     file(REAL_PATH ${project_path} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
-    if(NOT project_name)
-        o3de_read_json_key(project_name ${project_real_path}/project.json "project_name")
-        message(WARNING "The project located at path ${project_real_path} has a valid \"project name\" of '${project_name}' read from it's project.json file."
-            " This indicates that the ${project_real_path}/CMakeLists.txt is not properly appending the \"project name\" "
-            "to the LY_PROJECTS_TARGET_NAME global property. Other configuration errors might occur")
-    endif()
 
     ################################################################################
     # Assets
@@ -143,63 +137,56 @@ foreach(project_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJEC
     # Server
     ################################################################################
     if(PAL_TRAIT_BUILD_SERVER_SUPPORTED)
+        ly_add_target(
+            NAME ${project_name}.ServerLauncher APPLICATION
+            NAMESPACE AZ
+            FILES_CMAKE
+                ${CMAKE_CURRENT_LIST_DIR}/launcher_project_files.cmake
+            PLATFORM_INCLUDE_FILES
+                ${pal_dir}/launcher_project_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
+            COMPILE_DEFINITIONS
+                PRIVATE
+                    # Adds the name of the project/game
+                    LY_PROJECT_NAME="${project_name}"
+                    # Adds the ${project_name}_ServerLauncher target as a define so for the Settings Registry to use
+                    # when loading .setreg file specializations
+                    # This is needed so that only gems for the project server launcher are loaded
+                    LY_CMAKE_TARGET="${project_name}_ServerLauncher"
+            INCLUDE_DIRECTORIES
+                PRIVATE
+                    .
+                    ${CMAKE_CURRENT_BINARY_DIR}/${project_name}.ServerLauncher/Includes # required for StaticModules.inl
+            BUILD_DEPENDENCIES
+                PRIVATE
+                    AZ::Launcher.Static
+                    AZ::Launcher.Server.Static
+                    ${server_build_dependencies}
+            RUNTIME_DEPENDENCIES
+                ${server_runtime_dependencies}
+        )
+        # Needs to be set manually after ly_add_target to prevent the default location overriding it
+        set_target_properties(${project_name}.ServerLauncher
+            PROPERTIES
+                FOLDER ${project_name}
+                LY_PROJECT_NAME ${project_name}
+        )
 
-        get_property(server_projects GLOBAL PROPERTY LY_LAUNCHER_SERVER_PROJECTS)
-        if(${project_name} IN_LIST server_projects)
-
-            ly_add_target(
-                NAME ${project_name}.ServerLauncher APPLICATION
-                NAMESPACE AZ
-                FILES_CMAKE
-                    ${CMAKE_CURRENT_LIST_DIR}/launcher_project_files.cmake
-                PLATFORM_INCLUDE_FILES
-                    ${pal_dir}/launcher_project_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
-                COMPILE_DEFINITIONS
-                    PRIVATE
-                        # Adds the name of the project/game
-                        LY_PROJECT_NAME="${project_name}"
-                        # Adds the ${project_name}_ServerLauncher target as a define so for the Settings Registry to use
-                        # when loading .setreg file specializations
-                        # This is needed so that only gems for the project server launcher are loaded
-                        LY_CMAKE_TARGET="${project_name}_ServerLauncher"
-                INCLUDE_DIRECTORIES
-                    PRIVATE
-                        .
-                        ${CMAKE_CURRENT_BINARY_DIR}/${project_name}.ServerLauncher/Includes # required for StaticModules.inl
-                BUILD_DEPENDENCIES
-                    PRIVATE
-                        AZ::Launcher.Static
-                        AZ::Launcher.Server.Static
-                        ${server_build_dependencies}
-                RUNTIME_DEPENDENCIES
-                    ${server_runtime_dependencies}
-            )
-            # Needs to be set manually after ly_add_target to prevent the default location overriding it
-            set_target_properties(${project_name}.ServerLauncher
-                PROPERTIES 
-                    FOLDER ${project_name}
-                    LY_PROJECT_NAME ${project_name}
-            )
-
-            if(LY_DEFAULT_PROJECT_PATH)
-                if (TARGET ${project_name})
-                    get_target_property(project_server_launcher_additional_args ${project_name} SERVERLAUNCHER_ADDITIONAL_VS_DEBUGGER_COMMAND_ARGUMENTS)
-                    if (project_server_launcher_additional_args)
-                        # Avoid pushing param-NOTFOUND into the argument in case this property wasn't found
-                        set(additional_server_vs_debugger_args "${project_server_launcher_additional_args}")
-                    endif()
+        if(LY_DEFAULT_PROJECT_PATH)
+            if (TARGET ${project_name})
+                get_target_property(project_server_launcher_additional_args ${project_name} SERVERLAUNCHER_ADDITIONAL_VS_DEBUGGER_COMMAND_ARGUMENTS)
+                if (project_server_launcher_additional_args)
+                    # Avoid pushing param-NOTFOUND into the argument in case this property wasn't found
+                    set(additional_server_vs_debugger_args "${project_server_launcher_additional_args}")
                 endif()
-                
-                set_property(TARGET ${project_name}.ServerLauncher APPEND PROPERTY VS_DEBUGGER_COMMAND_ARGUMENTS 
-                    "--project-path=\"${LY_DEFAULT_PROJECT_PATH}\" ${additional_server_vs_debugger_args}")
             endif()
 
-            # Associate the Servers Gem Variant with each projects ServerLauncher
-            ly_set_gem_variant_to_load(TARGETS ${project_name}.ServerLauncher VARIANTS Servers)
+            set_property(TARGET ${project_name}.ServerLauncher APPEND PROPERTY VS_DEBUGGER_COMMAND_ARGUMENTS
+                "--project-path=\"${LY_DEFAULT_PROJECT_PATH}\" ${additional_server_vs_debugger_args}")
         endif()
 
+        # Associate the Servers Gem Variant with each projects ServerLauncher
+        ly_set_gem_variant_to_load(TARGETS ${project_name}.ServerLauncher VARIANTS Servers)
     endif()
-
 endforeach()
 
 #! Defer generation of the StaticModules.inl file needed in monolithic builds until after all the CMake targets are known

+ 1 - 2
Gems/Atom/Asset/ImageProcessingAtom/gem.json

@@ -15,7 +15,6 @@
     "documentation_url": "",
     "dependencies": [
         "Atom_RPI",
-        "Atom_RHI",
-        "Atom"
+        "Atom_RHI"
     ]
 }

+ 0 - 1
Gems/Atom/Feature/Common/gem.json

@@ -15,7 +15,6 @@
     "documentation_url": "",
     "dependencies": [
         "Atom_RPI",
-        "Atom",
         "ImGui",
         "Atom_RHI"
     ]

+ 0 - 1
Gems/Atom/RHI/DX12/gem.json

@@ -14,6 +14,5 @@
     "requirements": "",
     "documentation_url": "",
     "dependencies": [
-        "Atom_RHI"
     ]
 }

+ 0 - 1
Gems/Atom/RHI/Metal/gem.json

@@ -14,6 +14,5 @@
     "requirements": "",
     "documentation_url": "",
     "dependencies": [
-        "Atom_RHI"
     ]
 }

+ 0 - 1
Gems/Atom/RHI/Null/gem.json

@@ -14,6 +14,5 @@
     "requirements": "",
     "documentation_url": "",
     "dependencies": [
-        "Atom_RHI"
     ]
 }

+ 0 - 1
Gems/Atom/RHI/Vulkan/gem.json

@@ -14,6 +14,5 @@
     "requirements": "",
     "documentation_url": "",
     "dependencies": [
-        "Atom_RHI"
     ]
 }

+ 1 - 3
Gems/Atom/RHI/gem.json

@@ -22,8 +22,6 @@
         "Atom_RHI_DX12",
         "Atom_RHI_Metal",
         "Atom_RHI_Vulkan",
-        "Atom_RHI_Salem",
-        "Atom_RHI_Null",
-        "Atom_Feature_Common"
+        "Atom_RHI_Null"
     ]
 }

+ 0 - 1
Gems/AtomLyIntegration/AtomBridge/gem.json

@@ -26,7 +26,6 @@
         "ImguiAtom",
         "AtomFont",
         "AtomViewportDisplayInfo",
-        "Atom",
         "AtomShader",
         "ImageProcessingAtom",
         "AtomToolsFramework",

+ 1 - 2
Gems/AtomLyIntegration/AtomFont/gem.json

@@ -15,7 +15,6 @@
     "documentation_url": "",
     "dependencies": [
         "Atom_RHI",
-        "Atom_RPI",
-        "Atom_AtomBridge"
+        "Atom_RPI"
     ]
 }

+ 1 - 2
Gems/AtomLyIntegration/AtomImGuiTools/gem.json

@@ -17,7 +17,6 @@
     "requirements": "",
     "documentation_url": "",
     "dependencies": [
-        "ImguiAtom",
-        "Atom"
+        "ImguiAtom"
     ]
 }

+ 1 - 2
Gems/AtomLyIntegration/AtomViewportDisplayIcons/gem.json

@@ -16,7 +16,6 @@
     "dependencies": [
         "Atom_RHI",
         "Atom_RPI",
-        "Atom_Bootstrap",
-        "Atom_AtomBridge"
+        "Atom_Bootstrap"
     ]
 }

+ 0 - 1
Gems/AtomLyIntegration/EMotionFXAtom/gem.json

@@ -15,7 +15,6 @@
     "documentation_url": "",
     "dependencies": [
         "EMotionFX",
-        "Atom",
         "Atom_Feature_Common",
         "Atom_RPI",
         "Atom_RHI",

+ 0 - 11
Templates/DefaultProject/Template/CMakeLists.txt

@@ -18,15 +18,4 @@ if(NOT PROJECT_NAME)
     include(cmake/EngineFinder.cmake OPTIONAL)
     find_package(o3de REQUIRED)
     o3de_initialize()
-else()
-    # Add the project_name to global LY_PROJECTS_TARGET_NAME property
-    file(READ "${CMAKE_CURRENT_LIST_DIR}/project.json" project_json)
-
-    string(JSON project_target_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
-    if(json_error)
-        message(FATAL_ERROR "Unable to read key 'project_name' from 'project.json'")
-    endif()
-
-    set_property(GLOBAL APPEND PROPERTY LY_PROJECTS_TARGET_NAME ${project_target_name})
-
 endif()

+ 0 - 7
Templates/DefaultProject/Template/Gem/CMakeLists.txt

@@ -74,10 +74,3 @@ ly_create_alias(NAME ${Name}.Servers  NAMESPACE Gem TARGETS Gem::${Name})
 
 # Enable the specified list of gems from GEM_FILE or GEMS list for this specific project:
 ly_enable_gems(PROJECT_NAME ${Name} GEM_FILE enabled_gems.cmake)
-
-if(PAL_TRAIT_BUILD_SERVER_SUPPORTED)
-    # this property causes it to actually make a ServerLauncher.
-    # if you don't want a Server application, you can remove this and the
-    # following ly_enable_gems lines.
-    set_property(GLOBAL APPEND PROPERTY LY_LAUNCHER_SERVER_PROJECTS ${Name})
-endif()

+ 0 - 11
Templates/MinimalProject/Template/CMakeLists.txt

@@ -18,15 +18,4 @@ if(NOT PROJECT_NAME)
     include(cmake/EngineFinder.cmake OPTIONAL)
     find_package(o3de REQUIRED)
     o3de_initialize()
-else()
-    # Add the project_name to global LY_PROJECTS_TARGET_NAME property
-    file(READ "${CMAKE_CURRENT_LIST_DIR}/project.json" project_json)
-
-    string(JSON project_target_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
-    if(json_error)
-        message(FATAL_ERROR "Unable to read key 'project_name' from 'project.json'")
-    endif()
-
-    set_property(GLOBAL APPEND PROPERTY LY_PROJECTS_TARGET_NAME ${project_target_name})
-
 endif()

+ 0 - 8
Templates/MinimalProject/Template/Gem/CMakeLists.txt

@@ -74,11 +74,3 @@ ly_create_alias(NAME ${Name}.Servers  NAMESPACE Gem TARGETS Gem::${Name})
 
 # Enable the specified list of gems from GEM_FILE or GEMS list for this specific project:
 ly_enable_gems(PROJECT_NAME ${Name} GEM_FILE enabled_gems.cmake)
-
-
-if(PAL_TRAIT_BUILD_SERVER_SUPPORTED)
-    # this property causes it to actually make a ServerLauncher.
-    # if you don't want a Server application, you can remove this and the
-    # following ly_enable_gems lines.
-    set_property(GLOBAL APPEND PROPERTY LY_LAUNCHER_SERVER_PROJECTS ${Name})
-endif()

+ 2 - 2
Tools/LyTestTools/tests/CMakeLists.txt

@@ -16,8 +16,8 @@ ly_add_pytest(
     COMPONENT TestTools
 )
 
-get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
-if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS_TARGET_NAME)
+get_property(O3DE_PROJECTS_NAME GLOBAL PROPERTY O3DE_PROJECTS_NAME)
+if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST O3DE_PROJECTS_NAME)
     # Integration tests.
 
     ly_add_pytest(

+ 1 - 17
Tools/RemoteConsole/ly_remote_console/tests/CMakeLists.txt

@@ -12,20 +12,4 @@
 ly_add_pytest(
     NAME RemoteConsole_UnitTests_main_no_gpu
     PATH ${CMAKE_CURRENT_LIST_DIR}/unit/
-)
-
-get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
-if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS_TARGET_NAME)
-    # Integration tests.
-#    ly_add_pytest(
-#        NAME RemoteConsole_IntegTests_periodic_no_gpu
-#        PATH ${CMAKE_CURRENT_LIST_DIR}/integ/test_remote_console.py
-#        TEST_SERIAL
-#        TEST_SUITE periodic
-#        RUNTIME_DEPENDENCIES
-#            Legacy::Editor
-#            AssetProcessor
-#            AutomatedTesting.GameLauncher
-#            AutomatedTesting.Assets
-#    )
-endif()
+)

+ 2 - 2
cmake/FileUtil.cmake

@@ -112,7 +112,7 @@ endfunction()
 # the cmake project generation. This will provide a bridge to non-cmake tools to read the platform-specific cmake
 # project generation settings.
 #
-get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
+get_property(O3DE_PROJECTS_NAME GLOBAL PROPERTY O3DE_PROJECTS_NAME)
 function(ly_update_platform_settings)
     # Update the <platform>.last file to keep track of the recent build_dir
     set(ly_platform_last_path "${CMAKE_BINARY_DIR}/platform.settings")
@@ -122,7 +122,7 @@ function(ly_update_platform_settings)
 
 [settings]
 platform=${PAL_PLATFORM_NAME}
-game_projects=${LY_PROJECTS_TARGET_NAME}
+game_projects=${O3DE_PROJECTS_NAME}
 asset_deploy_mode=${LY_ASSET_DEPLOY_MODE}
 asset_deploy_type=${LY_ASSET_DEPLOY_ASSET_TYPE}
 override_pak_root=${LY_ASSET_OVERRIDE_PAK_FOLDER_ROOT}

+ 20 - 6
cmake/PAL.cmake

@@ -52,24 +52,38 @@ function(o3de_read_manifest o3de_manifest_json_data)
 endfunction()
 
 
-#! o3de_find_gem: returns the gem path
+#! o3de_find_gem_with_registered_external_subdirs: Query the path of a gem using its name
 #
 # \arg:gem_name the gem name to find
-# \arg:the path of the gem
-function(o3de_find_gem gem_name gem_path)
-    get_all_external_subdirectories(all_external_subdirs)
-    foreach(external_subdir IN LISTS all_external_subdirs)
+# \arg:output_gem_path the path of the gem to set
+# \arg:registered_external_subdirs a list of external subdirectories registered accross
+#      all manifest files to look for gems
+function(o3de_find_gem_with_registered_external_subdirs gem_name output_gem_path registered_external_subdirs)
+    foreach(external_subdir IN LISTS registered_external_subdirs)
         set(candidate_gem_path ${external_subdir}/gem.json)
         if(EXISTS ${candidate_gem_path})
             o3de_read_json_key(gem_json_name ${candidate_gem_path} "gem_name")
             if(gem_json_name STREQUAL gem_name)
-                set(${gem_path} ${external_subdir} PARENT_SCOPE)
+                set(${output_gem_path} ${external_subdir} PARENT_SCOPE)
                 return()
             endif()
         endif()
     endforeach()
 endfunction()
 
+#! o3de_find_gem: Query the path of a gem using its name
+#
+# \arg:gem_name the gem name to find
+# \arg:output_gem_path the path of the gem to set
+#
+# If the list of registered external subdirectories are available in the caller,
+# then slightly better more performance can be achieved by calling `o3de_find_gem_with_registered_external_subdirs` above
+function(o3de_find_gem gem_name output_gem_path)
+    get_all_external_subdirectories(registered_external_subdirs)
+    o3de_find_gem_with_registered_external_subdirs(${gem_name} gem_path "${registered_external_subdirs}")
+    set(${output_gem_path} ${gem_path} PARENT_SCOPE)
+endfunction()
+
 #! o3de_manifest_restricted: returns the manifests restricted paths
 #
 # \arg:restricted returns the restricted elements from the manifest

+ 5 - 5
cmake/Platform/Common/Install_common.cmake

@@ -447,10 +447,10 @@ function(ly_setup_cmake_install)
         )
     endforeach()
 
-    # Transform the LY_EXTERNAL_SUBDIRS global property list into a json array
+    # Transform the list of all external subdirectories used by the engine + projects into a json array
     set(indent "        ")
-    get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
-    list(REMOVE_DUPLICATES external_subdirs)
+
+    get_external_subdirectories_in_use(external_subdirs)
     foreach(external_subdir ${external_subdirs})
         # If an external subdirectory is not a subdirectory of the engine root, then
         # prepend "External" to its subdirectory root
@@ -670,8 +670,8 @@ function(ly_setup_assets)
     # the install layout from the root directory. Such as <external-subdirectory-root>/Cache.
     # This is also done to avoid globbing thousands of files in subdirectories that shouldn't
     # be processed.
-    get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
-    foreach(gem_candidate_dir IN LISTS external_subdirs LY_PROJECTS)
+    get_external_subdirectories_in_use(external_subdirs)
+    foreach(gem_candidate_dir IN LISTS external_subdirs)
         file(REAL_PATH ${gem_candidate_dir} gem_candidate_dir BASE_DIRECTORY ${LY_ROOT_FOLDER})
         # Don't recurse immediately in order to exclude transient source artifacts
         file(GLOB

+ 7 - 1
cmake/Projects.cmake

@@ -190,11 +190,17 @@ foreach(project ${LY_PROJECTS})
 
     cmake_path(GET project FILENAME project_folder_name )
     list(APPEND LY_PROJECTS_FOLDER_NAME ${project_folder_name})
-    add_subdirectory(${project} "${project_folder_name}-${full_directory_hash}")
+    # Generate a setreg file with the path to cmake binary directory
+    # into <project-path>/user/Registry/Platform/<Platform> directory
     ly_generate_project_build_path_setreg(${full_directory_path})
 
     # Get project name
     o3de_read_json_key(project_name ${full_directory_path}/project.json "project_name")
+
+    # Set the project name into the global O3DE_PROJECTS_NAME property
+    set_property(GLOBAL APPEND PROPERTY O3DE_PROJECTS_NAME ${project_name})
+
+    # Append the project external directory to LY_EXTERNAL_SUBDIR_${project_name} property
     add_project_json_external_subdirectories(${full_directory_path} "${project_name}")
 
     install_project_asset_artifacts(${full_directory_path})

+ 183 - 49
cmake/Subdirectories.cmake

@@ -12,25 +12,24 @@ include_guard()
 # Subdirectory processing
 ################################################################################
 
-# this function is building up the LY_EXTERNAL_SUBDIRS global property
+# The following functions is for gathering the list of external subdirectories
+# provided by the engine.json
 function(add_engine_gem_json_external_subdirectories gem_path)
     set(gem_json_path ${gem_path}/gem.json)
     if(EXISTS ${gem_json_path})
-        o3de_read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
         # Read the gem_name from the gem.json and map it to the gem path
         o3de_read_json_key(gem_name "${gem_path}/gem.json" "gem_name")
         if (gem_name)
             set_property(GLOBAL PROPERTY "@GEMROOT:${gem_name}@" "${gem_path}")
         endif()
 
-        foreach(gem_external_subdir ${gem_external_subdirs})
+        o3de_read_json_external_subdirs(gem_external_subdirs "${gem_path}/gem.json")
+        foreach(gem_external_subdir IN LISTS gem_external_subdirs)
             file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
 
-            # Append external subdirectory if it is not in global property
-            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_ENGINE PROPERTY
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE)
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
-                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir})
                 add_engine_gem_json_external_subdirectories(${real_external_subdir})
             endif()
@@ -42,14 +41,12 @@ function(add_engine_json_external_subdirectories)
     set(engine_json_path ${LY_ROOT_FOLDER}/engine.json)
     if(EXISTS ${engine_json_path})
         o3de_read_json_external_subdirs(engine_external_subdirs ${engine_json_path})
-        foreach(engine_external_subdir ${engine_external_subdirs})
+        foreach(engine_external_subdir IN LISTS engine_external_subdirs )
             file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER})
 
-            # Append external subdirectory if it is not in global property
-            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_ENGINE PROPERTY
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE)
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
-                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir})
                 add_engine_gem_json_external_subdirectories(${real_external_subdir})
             endif()
@@ -58,6 +55,8 @@ function(add_engine_json_external_subdirectories)
 endfunction()
 
 
+# The following functions is for gathering the list of external subdirectories
+# provided by the project.json
 function(add_project_gem_json_external_subdirectories gem_path project_name)
     set(gem_json_path ${gem_path}/gem.json)
     if(EXISTS ${gem_json_path})
@@ -68,14 +67,12 @@ function(add_project_gem_json_external_subdirectories gem_path project_name)
             set_property(GLOBAL PROPERTY "@GEMROOT:${gem_name}@" "${gem_path}")
         endif()
 
-        foreach(gem_external_subdir ${gem_external_subdirs})
+        foreach(gem_external_subdir IN LISTS gem_external_subdirs)
             file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
 
-            # Append external subdirectory if it is not in global property
-            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_${project_name} PROPERTY
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_${project_name})
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
-                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir})
                 add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}")
             endif()
@@ -87,14 +84,12 @@ function(add_project_json_external_subdirectories project_path project_name)
     set(project_json_path ${project_path}/project.json)
     if(EXISTS ${project_json_path})
         o3de_read_json_external_subdirs(project_external_subdirs ${project_path}/project.json)
-        foreach(project_external_subdir ${project_external_subdirs})
+        foreach(project_external_subdir IN LISTS project_external_subdirs)
             file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path})
 
-            # Append external subdirectory if it is not in global property
-            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_${project_name} PROPERTY
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_${project_name})
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
-                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir})
                 add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}")
             endif()
@@ -115,11 +110,10 @@ function(add_o3de_manifest_gem_json_external_subdirectories gem_path)
             set_property(GLOBAL PROPERTY "@GEMROOT:${gem_name}@" "${gem_path}")
         endif()
 
-        foreach(gem_external_subdir ${gem_external_subdirs})
+        foreach(gem_external_subdir IN LISTS gem_external_subdirs)
             file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
 
             # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY
-            # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project
             get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir})
@@ -135,11 +129,10 @@ function(add_o3de_manifest_json_external_subdirectories)
     o3de_get_manifest_path(manifest_path)
     if(EXISTS ${manifest_path})
         o3de_read_json_external_subdirs(o3de_manifest_external_subdirs ${manifest_path})
-        foreach(manifest_external_subdir ${o3de_manifest_external_subdirs})
+        foreach(manifest_external_subdir IN LISTS o3de_manifest_external_subdirs)
             file(REAL_PATH ${manifest_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER})
 
             # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY
-            # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project
             get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
             if(NOT real_external_subdir IN_LIST current_external_subdirs)
                 set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir})
@@ -152,54 +145,195 @@ endfunction()
 #! Gather unique_list of all external subdirectories that is union
 #! of the engine.json, project.json, o3de_manifest.json and any gem.json files found visiting
 function(get_all_external_subdirectories output_subdirs)
+    # Gather user supplied external subdirectories via the Cache Variable
     get_property(all_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
     get_property(manifest_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
     list(APPEND all_external_subdirs ${manifest_external_subdirs})
+    get_property(engine_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE)
+    list(APPEND all_external_subdirs ${engine_external_subdirs})
+
+    # Gather the list of every configured project external subdirectory
+    # and and append them to the list of external subdirectories
+    get_property(project_names GLOBAL PROPERTY O3DE_PROJECTS_NAME)
+    foreach(project_name IN LISTS project_names)
+        get_property(project_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_${project_name})
+        list(APPEND all_external_subdirs ${project_external_subdirs})
+    endforeach()
+
     list(REMOVE_DUPLICATES all_external_subdirs)
     set(${output_subdirs} ${all_external_subdirs} PARENT_SCOPE)
 endfunction()
 
-#! add_registered_gems_to_external_subdirs:
-#! Accepts a list of gem_names (which can be read from the project.json or engine.json)
+
+#! Accepts a list of gem names (which can be read from the project.json, gem.json or engine.json)
+#! and a list of ALL registered external subdirectories across all manifest
 #! and cross checks them against union of all external subdirectories to determine the gem path.
-#! If that gem exist it is appended to LY_EXTERNAL_SUBDIRS so that that the build generator
-#! adds to the generated build project.
-#! Otherwise a fatal error is logged indicating that is not gem could not be found in the list of external subdirectories
-function(add_registered_gems_to_external_subdirs gem_names)
+#! If that gem path exist it is appended to the output parameter output gem directories parameter
+#! A fatal error is logged indicating that is not gem could not be found in the list of external subdirectories
+function(query_gem_paths_from_external_subdirs output_gem_dirs gem_names registered_external_subdirs)
     if (gem_names)
-        get_all_external_subdirectories(all_external_subdirs)
         foreach(gem_name IN LISTS gem_names)
             unset(gem_path)
-            o3de_find_gem(${gem_name} gem_path)
+            o3de_find_gem_with_registered_external_subdirs(${gem_name} gem_path "${registered_external_subdirs}")
             if (gem_path)
-                set_property(GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS ${gem_path} APPEND)
+                list(APPEND gem_dirs ${gem_path})
             else()
-                list(JOIN all_external_subdirs "\n" external_subdirs_formatted)
-                message(SEND_ERROR "The gem \"${gem_name}\" from the \"gem_names\" field in the engine.json/project.json "
+                list(JOIN registered_external_subdirs "\n" external_subdirs_formatted)
+                message(SEND_ERROR "The gem \"${gem_name}\""
                 " could not be found in any gem.json from the following list of registered external subdirectories:\n"
                 "${external_subdirs_formatted}")
                 break()
             endif()
         endforeach()
     endif()
+    set(${output_gem_dirs} ${gem_dirs} PARENT_SCOPE)
 endfunction()
 
-function(add_subdirectory_on_external_subdirs)
-    # Lookup the paths of "gem_names" array all project.json files and engine.json
-    # and append them to the LY_EXTERNAL_SUBDIRS property
-    foreach(project ${LY_PROJECTS})
-        file(REAL_PATH ${project} full_directory_path BASE_DIRECTORY ${CMAKE_SOURCE_DIR})
-        o3de_read_json_array(gem_names ${full_directory_path}/project.json "gem_names")
-        add_registered_gems_to_external_subdirs("${gem_names}")
+#! Queries the list of gem names against the list of ALL registered external subdirectories
+#! in order to determine the paths corresponding to the gem names
+function(add_registered_gems_to_external_subdirs output_gem_dirs gem_names)
+    get_all_external_subdirectories(registered_external_subdirs)
+    query_gem_paths_from_external_subdirs(gem_dirs "${gem_names}" "${registered_external_subdirs}")
+    set(${output_gem_dirs} ${gem_dirs} PARENT_SCOPE)
+endfunction()
+
+#! Recurses "dependencies" array if the external subdirectory is a gem(contains a gem.json)
+#! for each subdirectory in use.
+#! This function looks up the each dependent gem path from the registered external subdirectory set
+#! That list of resolved gem paths then have this function invoked on it to perform the same behavior
+#! When every descendent gem referenced from the "dependencies" field of the current subdirectory is visited
+#! it is then appended to a list of output external subdirectories
+#! NOTE: This must be invoked after all the add_*_json_external_subdirectories function
+function(reorder_dependent_gems_with_cycle_detection _output_external_dirs subdirs_in_use registered_external_subdirs cycle_detection_set)
+    # output_external_dirs is a variable whose value is the name of a variable to set in the parent scope
+    # So double resolve the variable to retrieve its value
+    set(current_external_dirs "${${_output_external_dirs}}")
+
+    foreach(external_subdir IN LISTS subdirs_in_use)
+        # If a cycle is detected, fatal error and output the list of subdirectories that led to the outcome
+        if (external_subdir IN_LIST cycle_detection_set)
+            message(FATAL_ERROR "While visiting \"${external_subdir}\", a cycle was detected in the \"dependencies\""
+            " array of the following gem.json files in the directories: ${cycle_detection_set}")
+        endif()
+        # This subdirectory has already been processed so skip to the next one
+        if(external_subdir IN_LIST current_external_dirs)
+            continue()
+        endif()
+
+        get_property(ordered_dependent_subdirs GLOBAL PROPERTY "Dependent:${external_subdir}")
+        if(ordered_dependent_subdirs)
+            # Re-use the cached list of dependent subdirs if available
+            list(APPEND current_external_dirs "${ordered_dependent_subdirs}")
+        else()
+            cmake_path(SET gem_manifest_path "${external_subdir}/gem.json")
+            if(EXISTS ${gem_manifest_path})
+                # Read the "dependencies" array from gem.json
+                o3de_read_json_array(dependencies_array "${gem_manifest_path}" "dependencies")
+                # Lookup the paths using the dependent gem names
+                unset(reference_external_dirs)
+                query_gem_paths_from_external_subdirs(reference_external_dirs "${dependencies_array}" "${registered_external_subdirs}")
+
+                # Append the external subdirectory into the children cycle_detection_set
+                set(child_cycle_detection_set ${cycle_detection_set} ${external_subdir})
+
+                # Recursively visit the list of gem dependencies for the current external subdir
+                reorder_dependent_gems_with_cycle_detection(current_external_dirs "${reference_external_dirs}"
+                    "${registered_external_subdirs}" "${child_cycle_detection_set}")
+                # Append the referenced gem directories before the current external subdir so that they are visited first
+                list(APPEND current_external_dirs "${reference_external_dirs}")
+
+                # Cache the list of external subdirectories so that it can be reused in subsequent calls
+                set_property(GLOBAL PROPERTY "Dependent:${external_subdir}" "${reference_external_dirs}")
+            endif()
+        endif()
+
+        # Now append the external subdir
+        list(APPEND current_external_dirs ${external_subdir})
     endforeach()
+
+    set(${_output_external_dirs} ${current_external_dirs} PARENT_SCOPE)
+endfunction()
+
+function(reorder_dependent_gems_before_external_subdirs output_gem_subdirs subdirs_in_use)
+    # Lookup the registered external subdirectories once and re-use it for each call
+    get_all_external_subdirectories(registered_external_subdirs)
+    # Supply an empty visited set and cycle_detection_set argument
+    reorder_dependent_gems_with_cycle_detection(output_external_dirs "${subdirs_in_use}" "${registered_external_subdirs}" "")
+    set(${output_gem_subdirs} ${output_external_dirs} PARENT_SCOPE)
+endfunction()
+
+#! Gather unique_list of all external subdirectories that the project provides or uses
+#! The list is made up of the following
+#! - The paths of gems referenced in the project.json "gem_names" key. Those paths are queried
+#!   from the "external_subdirectories" in o3de_manifest.json
+#! - The project path
+#! - The list of external_subdirectories found by recursively visting the project.json "external_subdirectories"
+function(get_all_external_subdirectories_for_project output_subdirs project_name project_path)
+    # Append the gems referenced by name from "gem_names" field in the project.json
+    # These gems are registered in the users o3de_manifest.json
+    o3de_read_json_array(gem_names ${project_path}/project.json "gem_names")
+    add_registered_gems_to_external_subdirs(project_gem_reference_dirs "${gem_names}")
+    list(APPEND subdirs_for_project ${project_gem_reference_dirs})
+
+    # Append the project root path to the list of external subdirectories so that it is visited
+    list(APPEND subdirs_for_project ${project_path})
+
+    # Append the list of external_subdirectories that come with the project
+    get_property(project_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_${project_name})
+    list(APPEND subdirs_for_project ${project_external_subdirs})
+
+    list(REMOVE_DUPLICATES subdirs_for_project)
+    set(${output_subdirs} ${subdirs_for_project} PARENT_SCOPE)
+endfunction()
+
+#! Gather the unqiue list of all external subdirectories that the engine provides("external_subdirectories")
+#! plus all external subdirectories that every active project provides("external_subdirectories")
+#! or references("gem_names")
+function(get_external_subdirectories_in_use output_subdirs)
+    # Gather the list of external subdirectories set through the LY_EXTERNAL_SUBDIRS Cache Variable
+    get_property(all_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+    # Append the list of Extenal Subdirectories from the engine.json
+    get_property(engine_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE)
+    list(APPEND all_external_subdirs ${engine_external_subdirs})
+    # Append the gems referenced by name from "gem_names" field in the engine.json
+    # These gems are registered in the users o3de_manifest.json
     o3de_read_json_array(gem_names ${LY_ROOT_FOLDER}/engine.json "gem_names")
-    add_registered_gems_to_external_subdirs("${gem_names}")
+    add_registered_gems_to_external_subdirs(engine_gem_reference_dirs "${gem_names}")
+    list(APPEND all_external_subdirs ${engine_gem_reference_dirs})
+
+    # Visit each LY_PROJECTS entry and append the external subdirectories
+    # the project provides and references
+    get_property(O3DE_PROJECTS_NAME GLOBAL PROPERTY O3DE_PROJECTS_NAME)
+    foreach(project_name project_path IN ZIP_LISTS O3DE_PROJECTS_NAME LY_PROJECTS)
+        file(REAL_PATH ${project_path} full_directory_path BASE_DIRECTORY ${CMAKE_SOURCE_DIR})
+        get_all_external_subdirectories_for_project(external_subdirs ${project_name} ${full_directory_path} )
+        list(APPEND all_external_subdirs ${external_subdirs})
+    endforeach()
+
+    # Make sure any gems in the "dependencies" field of a gem.json
+    # are ordered before that gem, so they are parsed first.
+    reorder_dependent_gems_before_external_subdirs(all_external_subdirs "${all_external_subdirs}")
+    list(REMOVE_DUPLICATES all_external_subdirs)
+    set(${output_subdirs} ${all_external_subdirs} PARENT_SCOPE)
+endfunction()
+
+#! Visit all external subdirectories that is in use by the engine and each project
+#! This visits "external_subdirectories" listed in the engine.json,
+#! the "external_subdirectories" listed in the each LY_PROJECTS project.json,
+#! and the "external_subdirectories" listed o3de_manifest.json in which the engine.json/project.json
+#! references in their "gem_names" key.
+function(add_subdirectory_on_external_subdirs)
+    # Query the list of external subdirectories in use by the engine and any projects
+    get_external_subdirectories_in_use(all_external_subdirs)
+
+    # Log the external subdirectory visit order
+    message(VERBOSE "add_subdirectory will be called on the following external subdirectories in order:")
+    foreach(external_directory IN LISTS all_external_subdirs)
+        message(VERBOSE "${external_directory}")
+    endforeach()
 
-    get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
-    list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs})
-    list(REMOVE_DUPLICATES LY_EXTERNAL_SUBDIRS)
     # Loop over the additional external subdirectories and invoke add_subdirectory on them
-    foreach(external_directory ${LY_EXTERNAL_SUBDIRS})
+    foreach(external_directory IN LISTS all_external_subdirs)
         # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory
         # This is to deal with potential situations where multiple external directories has the same last directory name
         # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory