Implemented Support to allow project's to reference gems via the gem name (#7109)

* Implemented Support to allow project's to reference gems via the gem name

Updated the enable-gem command to add the name of the enabled gem to the "gem_names" array in the project.json
Updated the enable-gem test to validate this functionality

Centralized the CMake logic for locating external subdirectories to the Subdirectories.cmake script

Added an option to the edit-project-properties and edit-engine-properties o3de.py commands to add/remove/replace the "gem_names" field in the project.json and engine.json respectively

Added a CMake function to determine the root CMake "subdirectory" of any input path which is a parent of it.
This logic has been used to improve the installation of external gems to the <install-root>/External directory.

Tested out the install layout before submitting PR

fixes #7108

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Fixed the enable-gem test on Linux to resolve the mock path.

Renamed all of the o3de python test from "unit_test*.py" to "test*.py" to faciliate the python unittest module picking up the test automatically.

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Adding test for the disable_gem command.

Fixed some typos in engine_properties.py scrip.

Signed-off-by: lumberyard-employee-dm <[email protected]>

CMakeLists.txt

@@ -44,51 +44,11 @@ include(cmake/SettingsRegistry.cmake)
 include(cmake/TestImpactFramework/LYTestImpactFramework.cmake)
 include(cmake/CMakeFiles.cmake)
 include(cmake/O3DEJson.cmake)
+include(cmake/Subdirectories.cmake)
 
-################################################################################
-# Subdirectory processing
-################################################################################
-
-# this function is building up the LY_EXTERNAL_SUBDIRS global property
-function(add_engine_gem_json_external_subdirectories gem_path)
-    set(gem_json_path ${gem_path}/gem.json)
-    if(EXISTS ${gem_json_path})
-        read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
-        foreach(gem_external_subdir ${gem_external_subdirs})
-            file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
-            set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-            add_engine_gem_json_external_subdirectories(${real_external_subdir})
-        endforeach()
-    endif()
-endfunction()
-
-function(add_engine_json_external_subdirectories)
-    read_json_external_subdirs(engine_external_subdirs ${LY_ROOT_FOLDER}/engine.json)
-    foreach(engine_external_subdir ${engine_external_subdirs})
-        file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER})
-        set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-        add_engine_gem_json_external_subdirectories(${real_external_subdir})
-    endforeach()
-endfunction()
-
-function(add_subdirectory_on_externalsubdirs)
-    get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
-    list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs})
-    # Loop over the additional external subdirectories and invoke add_subdirectory on them
-    foreach(external_directory ${LY_EXTERNAL_SUBDIRS})
-        # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory
-        # This is to deal with potential situations where multiple external directories has the same last directory name
-        # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory
-        file(REAL_PATH ${external_directory} full_directory_path)
-        string(SHA256 full_directory_hash ${full_directory_path})
-        # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit
-        # when the external subdirectory contains relative paths of significant length
-        string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash)
-        # Use the last directory as the suffix path to use for the Binary Directory
-        get_filename_component(directory_name ${external_directory} NAME)
-        add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash})
-    endforeach()
-endfunction()
+# Gather the list of o3de_manifest external Subdirectories
+# into the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST_PROPERTY
+add_o3de_manifest_json_external_subdirectories()
 
 # Add the projects first so the Launcher can find them
 include(cmake/Projects.cmake)
@@ -99,9 +59,11 @@ endif()
 
 if(NOT INSTALLED_ENGINE)
     # Add external subdirectories listed in the engine.json. LY_EXTERNAL_SUBDIRS is a cache variable so the user can add extra
-    # external subdirectories. This should go before adding the rest of the targets so the targets are availbe to the launcher.
+    # external subdirectories. This should go before adding the rest of the targets so the targets are available to the launcher.
     add_engine_json_external_subdirectories()
-    add_subdirectory_on_externalsubdirs()
+
+    # Invoke add_subdirectory on external subdirectories that should be used a this point
+    add_subdirectory_on_external_subdirs()
 
     # Add the rest of the targets
     add_subdirectory(Assets)
@@ -114,7 +76,7 @@ if(NOT INSTALLED_ENGINE)
 
 else()
     ly_find_o3de_packages()
-    add_subdirectory_on_externalsubdirs()
+    add_subdirectory_on_external_subdirs()
 endif()
 
 ################################################################################

Gems/PhysXDebug/Code/CMakeLists.txt

@@ -10,7 +10,7 @@
 o3de_find_gem("PhysX" physx_gem_path)
 set(physx_gem_json ${physx_gem_path}/gem.json)
 o3de_restricted_path(${physx_gem_json} physx_gem_restricted_path physx_gem_parent_relative_path)
-o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} ${physx_gem_restricted_path} ${physx_gem_path} ${physx_gem_parent_relative_path})
+o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} "${physx_gem_restricted_path}" "${physx_gem_path}" "${physx_gem_parent_relative_path}")
 
 include(${physx_pal_source_dir}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) # for PAL_TRAIT_PHYSX_SUPPORTED
 

cmake/FileUtil.cmake

@@ -153,6 +153,36 @@ function(ly_get_last_path_segment_concat_sha256 absolute_path output_path)
     set(${output_path} ${last_path_segment_sha256_path} PARENT_SCOPE)
 endfunction()
 
+#! ly_get_root_subdirectory_which_is_parent: Locates the root source directory added the input directory
+#  as a subdirectory of the build, which an actual prefix of the input directory
+#  This is done by recursing through the PARENT_DIRECTORY "DIRECTORY" property
+#  The use for this is to locate the top most directory which called add_subdirectory from any input path
+#  i.e Given an
+#  LY_ROOT_FOLDER = D:\o3de
+#  EXTERNAL_SUBDIRS = [D:\TestGem, D:\o3de\Gems\MyGem]
+#  The LY_ROOT_FOLDER is responsible for invoking add_subdirectory on the external subdirectories
+#  so it in the PARENT_DIRECTORY property, of the subdirectory, though it might not be an actual "parent"
+#  If the input path to this function is D:\TestGem\Code, then the return value is D:\TestGem
+#  If the input path to this function is D:\o3de\Gems\MyGem, then the return value is D:\o3de
+
+# \arg:absolute_path - directory to locate top most parent "subdirectory", which is an "parent" of the input
+# \return:output_path- top most parent subdirectory, which is actual parent(i.e a prefix)
+function(ly_get_root_subdirectory_which_is_parent absolute_path output_path)
+    # Walk up the parent add_subdirectory calls until a parent directory which is not a prefix of the target directory
+    # is found
+    cmake_path(SET candidate_path ${absolute_path})
+    get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY)
+    cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir)
+    while(parent_subdir AND is_parent_subdir)
+        cmake_path(SET candidate_path "${parent_subdir}")
+        get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY)
+        cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir)
+    endwhile()
+
+    message(DEBUG "Root subdirectory of path \"${absolute_path}\" is \"${candidate_path}\"")
+    set(${output_path} ${candidate_path} PARENT_SCOPE)
+endfunction()
+
 #! ly_get_engine_relative_source_dir: Attempts to form a path relative to the BASE_DIRECTORY.
 #  If that fails the last path segment of the absolute_target_source_dir concatenated with a SHA256 hash to form a target directory
 # \arg:BASE_DIRECTORY - Directory to base relative path against. Defaults to LY_ROOT_FOLDER
@@ -167,14 +197,16 @@ function(ly_get_engine_relative_source_dir absolute_target_source_dir output_sou
     endif()
 
     # Get a relative target source directory to the LY root folder if possible
-    # Otherwise use the final component name
+    # Otherwise use the top most source directory which led to calling add_subdirectory on the input directory
+    ly_get_root_subdirectory_which_is_parent(${absolute_target_source_dir} root_subdir_of_target)
+    cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_source_dir)
+
     cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_source_dir_subdirectory_of_engine)
-    if(is_target_source_dir_subdirectory_of_engine)
-        cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir)
-    else()
-        ly_get_last_path_segment_concat_sha256(${absolute_target_source_dir} target_source_dir_last_path_segment)
+    if(NOT is_target_source_dir_subdirectory_of_engine)
+        cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname)
+        set(relative_subdir ${relative_target_source_dir})
         unset(relative_target_source_dir)
-        cmake_path(APPEND relative_target_source_dir "External" ${target_source_dir_last_path_segment})
+        cmake_path(APPEND relative_target_source_dir "External" ${root_subdir_dirname} ${relative_subdir})
     endif()
 
     set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE)

cmake/PAL.cmake

@@ -51,49 +51,21 @@ function(o3de_read_manifest o3de_manifest_json_data)
     endif()
 endfunction()
 
-#! o3de_recurse_gems: returns the gem paths
-#
-# \arg:object json path
-# \arg:gems returns the gems from the external subdirectory elements from the manifest
-function(o3de_recurse_gems object_json_path gems)
-    get_filename_component(object_json_parent_path ${object_json_path} DIRECTORY)
-    ly_file_read(${object_json_path} json_data)
-    string(JSON external_subdirectories_count ERROR_VARIABLE json_error LENGTH ${json_data} "external_subdirectories")
-    if(NOT json_error)
-        if(external_subdirectories_count GREATER 0)
-            math(EXPR external_subdirectories_range "${external_subdirectories_count}-1")
-            foreach(external_subdirectories_index RANGE ${external_subdirectories_range})
-                string(JSON external_subdirectories_entry ERROR_VARIABLE json_error GET ${json_data} "external_subdirectories" "${external_subdirectories_index}")
-                cmake_path(IS_RELATIVE external_subdirectories_entry is_relative)
-                if(${is_relative})
-                    cmake_path(ABSOLUTE_PATH external_subdirectories_entry BASE_DIRECTORY ${object_json_parent_path} NORMALIZE OUTPUT_VARIABLE external_subdirectories_entry)
-                endif()
-                if(EXISTS ${external_subdirectories_entry}/gem.json)
-                    list(APPEND gem_entries ${external_subdirectories_entry})
-                    o3de_recurse_gems(${external_subdirectories_entry}/gem.json gem_entries)
-                endif()
-            endforeach()
-        endif()
-    endif()
-    set(${gems} ${gem_entries} PARENT_SCOPE)
-endfunction()
 
 #! o3de_find_gem: returns the gem path
 #
 # \arg:gem_name the gem name to find
 # \arg:the path of the gem
 function(o3de_find_gem gem_name gem_path)
-    o3de_get_manifest_path(manifest_path)
-    if(EXISTS ${manifest_path})
-        o3de_recurse_gems(${manifest_path} gems)
-    endif()
-    o3de_recurse_gems(${LY_ROOT_FOLDER}/engine.json gems)
-    foreach(gem ${gems})
-        ly_file_read(${gem}/gem.json json_data)
-        string(JSON gem_json_name ERROR_VARIABLE json_error GET ${json_data} "gem_name")
-        if(gem_json_name STREQUAL gem_name)
-            set(${gem_path} ${gem} PARENT_SCOPE)
-            return()
+    get_all_external_subdirectories(all_external_subdirs)
+    foreach(external_subdir IN LISTS all_external_subdirs)
+        set(candidate_gem_path ${external_subdir}/gem.json)
+        if(EXISTS ${candidate_gem_path})
+            o3de_read_json_key(gem_json_name ${candidate_gem_path} "gem_name")
+            if(gem_json_name STREQUAL gem_name)
+                set(${gem_path} ${external_subdir} PARENT_SCOPE)
+                return()
+            endif()
         endif()
     endforeach()
 endfunction()

cmake/Platform/Common/Install_common.cmake

@@ -448,9 +448,27 @@ function(ly_setup_cmake_install)
     # Transform the LY_EXTERNAL_SUBDIRS global property list into a json array
     set(indent "        ")
     get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+    list(REMOVE_DUPLICATES external_subdirs)
     foreach(external_subdir ${external_subdirs})
-        cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE engine_rel_external_subdir)
-        list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"")
+        # If an external subdirectory is not a subdirectory of the engine root, then
+        # prepend "External" to its subdirectory root
+        ly_get_root_subdirectory_which_is_parent(${external_subdir} root_subdir_of_external_subdir)
+        cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${root_subdir_of_external_subdir} OUTPUT_VARIABLE engine_rel_external_subdir)
+
+        cmake_path(IS_PREFIX LY_ROOT_FOLDER ${external_subdir} is_subdirectory_of_engine)
+        if(NOT is_subdirectory_of_engine)
+            cmake_path(GET root_subdir_of_external_subdir FILENAME root_subdir_dirname)
+            set(relative_subdir ${engine_rel_external_subdir})
+            unset(engine_rel_external_subdir)
+            cmake_path(APPEND engine_rel_external_subdir "External" ${root_subdir_dirname} ${relative_subdir})
+        endif()
+
+        set(quoted_engine_rel_external_subdir "\"${engine_rel_external_subdir}\"")
+        if (quoted_engine_rel_external_subdir IN_LIST relative_external_subdirs)
+            message(WARNING "An external subdirectory \"${external_subdir}\" has been found twice when generating the engine.json for the install layout")
+        else()
+            list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"")
+        endif()
     endforeach()
     list(JOIN relative_external_subdirs ",\n${indent}" LY_INSTALL_EXTERNAL_SUBDIRS)
 
@@ -507,7 +525,17 @@ function(ly_setup_cmake_install)
     # Add to find_subdirectories all directories in which ly_add_target were called in
     get_property(all_subdirectories GLOBAL PROPERTY LY_ALL_TARGET_DIRECTORIES)
     foreach(target_subdirectory IN LISTS all_subdirectories)
-        cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_subdirectory)
+        ly_get_root_subdirectory_which_is_parent(${target_subdirectory} root_subdir_of_target)
+        cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_subdirectory)
+
+        cmake_path(IS_PREFIX LY_ROOT_FOLDER ${target_subdirectory} is_subdirectory_of_engine)
+        if(NOT is_subdirectory_of_engine)
+            cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname)
+            set(relative_subdir ${relative_target_subdirectory})
+            unset(relative_target_subdirectory)
+            cmake_path(APPEND relative_target_subdirectory "External" ${root_subdir_dirname} ${relative_subdir})
+        endif()
+
         string(APPEND find_subdirectories "add_subdirectory(${relative_target_subdirectory})\n")
     endforeach()
     set(permutation_find_subdirectories ${CMAKE_CURRENT_BINARY_DIR}/cmake/Platform/${PAL_PLATFORM_NAME}/${LY_BUILD_PERMUTATION}/o3de_subdirectories_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
@@ -657,12 +685,12 @@ function(ly_setup_assets)
         set_property(GLOBAL APPEND PROPERTY global_gem_candidate_dirs_prop ${gem_candidate_dir})
     endforeach()
 
-    # Iterate over each gem candidate directories and read populate a directory property
+    # Iterate over each gem candidate directories and populate a directory property
     # containing the files to copy over
     get_property(gem_candidate_dirs GLOBAL PROPERTY global_gem_candidate_dirs_prop)
     foreach(gem_candidate_dir IN LISTS gem_candidate_dirs)
         get_property(filtered_asset_paths DIRECTORY ${gem_candidate_dir} PROPERTY directory_filtered_asset_paths)
-        ly_get_last_path_segment_concat_sha256(${gem_candidate_dir} last_gem_root_path_segment)
+
         # Check if the gem is a subdirectory of the engine
         cmake_path(IS_PREFIX LY_ROOT_FOLDER ${gem_candidate_dir} is_gem_subdirectory_of_engine)
         
@@ -697,15 +725,16 @@ function(ly_setup_assets)
         # gem directories and files to install
         get_property(gems_assets_paths DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths)
         foreach(gem_absolute_path IN LISTS gems_assets_paths)
-            if(is_gem_subdirectory_of_engine)
-                cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir)
-            else()
-                # The gem resides outside of the LY_ROOT_FOLDER, so the destination is made relative to the
-                # gem candidate directory and placed under the "External" directory"
-                # directory
-                cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${gem_candidate_dir} OUTPUT_VARIABLE gem_relative_path)
+            # If an external subdirectory is not a subdirectory of the engine root, then
+            # prepend "External" to its subdirectory root
+            ly_get_root_subdirectory_which_is_parent(${gem_candidate_dir} root_subdir_of_gem)
+            cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${root_subdir_of_gem} OUTPUT_VARIABLE gem_install_dest_dir)
+
+            if(NOT is_gem_subdirectory_of_engine)
+                cmake_path(GET root_subdir_of_gem FILENAME root_subdir_dirname)
+                set(relative_subdir ${gem_install_dest_dir})
                 unset(gem_install_dest_dir)
-                cmake_path(APPEND gem_install_dest_dir "External" ${last_gem_root_path_segment} ${gem_relative_path})
+                cmake_path(APPEND gem_install_dest_dir "External" ${root_subdir_dirname} ${relative_subdir})
             endif()
 
             cmake_path(GET gem_install_dest_dir PARENT_PATH gem_install_dest_dir)

cmake/Projects.cmake

@@ -118,30 +118,6 @@ function(ly_generate_project_build_path_setreg project_real_path)
     file(GENERATE OUTPUT ${project_user_build_path_setreg_file} CONTENT ${project_build_path_setreg_content})
 endfunction()
 
-function(add_gem_json_external_subdirectories gem_path)
-    set(gem_json_path ${gem_path}/gem.json)
-    if(EXISTS ${gem_json_path})
-        read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
-        foreach(gem_external_subdir ${gem_external_subdirs})
-            file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
-            set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-            add_gem_json_external_subdirectories(${real_external_subdir})
-        endforeach()
-    endif()
-endfunction()
-
-function(add_project_json_external_subdirectories project_path)
-    set(project_json_path ${project_path}/project.json)
-    if(EXISTS ${project_json_path})
-        read_json_external_subdirs(project_external_subdirs ${project_path}/project.json)
-        foreach(project_external_subdir ${project_external_subdirs})
-            file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path})
-            set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
-            add_gem_json_external_subdirectories(${real_external_subdir})
-        endforeach()
-    endif()
-endfunction()
-
 function(install_project_asset_artifacts project_real_path)
     # The cmake tar command has a bit of a flaw
     # Any paths within the archive files it creates are relative to the current working directory.
@@ -212,16 +188,16 @@ foreach(project ${LY_PROJECTS})
     # when the external subdirectory contains relative paths of significant length
     string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash)
 
-    get_filename_component(project_folder_name ${project} NAME)
+    cmake_path(GET project FILENAME project_folder_name )
     list(APPEND LY_PROJECTS_FOLDER_NAME ${project_folder_name})
     add_subdirectory(${project} "${project_folder_name}-${full_directory_hash}")
     ly_generate_project_build_path_setreg(${full_directory_path})
-    add_project_json_external_subdirectories(${full_directory_path})
 
     # Get project name
     o3de_read_json_key(project_name ${full_directory_path}/project.json "project_name")
+    add_project_json_external_subdirectories(${full_directory_path} "${project_name}")
 
-   install_project_asset_artifacts(${full_directory_path})
+    install_project_asset_artifacts(${full_directory_path})
 
 endforeach()
 

cmake/Subdirectories.cmake

@@ -0,0 +1,197 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+include_guard()
+
+################################################################################
+# Subdirectory processing
+################################################################################
+
+# this function is building up the LY_EXTERNAL_SUBDIRS global property
+function(add_engine_gem_json_external_subdirectories gem_path)
+    set(gem_json_path ${gem_path}/gem.json)
+    if(EXISTS ${gem_json_path})
+        read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
+        foreach(gem_external_subdir ${gem_external_subdirs})
+            file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
+
+            # Append external subdirectory if it is not in global property
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
+                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir})
+                add_engine_gem_json_external_subdirectories(${real_external_subdir})
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+function(add_engine_json_external_subdirectories)
+    set(engine_json_path ${LY_ROOT_FOLDER}/engine.json)
+    if(EXISTS ${engine_json_path})
+        read_json_external_subdirs(engine_external_subdirs ${engine_json_path})
+        foreach(engine_external_subdir ${engine_external_subdirs})
+            file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER})
+
+            # Append external subdirectory if it is not in global property
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
+                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir})
+                add_engine_gem_json_external_subdirectories(${real_external_subdir})
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+
+function(add_project_gem_json_external_subdirectories gem_path project_name)
+    set(gem_json_path ${gem_path}/gem.json)
+    if(EXISTS ${gem_json_path})
+        read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
+        foreach(gem_external_subdir ${gem_external_subdirs})
+            file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
+
+            # Append external subdirectory if it is not in global property
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
+                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir})
+                add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}")
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+function(add_project_json_external_subdirectories project_path project_name)
+    set(project_json_path ${project_path}/project.json)
+    if(EXISTS ${project_json_path})
+        read_json_external_subdirs(project_external_subdirs ${project_path}/project.json)
+        foreach(project_external_subdir ${project_external_subdirs})
+            file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path})
+
+            # Append external subdirectory if it is not in global property
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir})
+                # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir})
+                add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}")
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+
+#! add_o3de_manifest_gem_json_external_subdirectories : Recurses through external subdirectories
+#! originally found in the add_o3de_manifest_json_external_subdirectories command
+function(add_o3de_manifest_gem_json_external_subdirectories gem_path)
+    set(gem_json_path ${gem_path}/gem.json)
+    if(EXISTS ${gem_json_path})
+        read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json)
+        foreach(gem_external_subdir ${gem_external_subdirs})
+            file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path})
+
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY
+            # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir})
+                add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir})
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+#! add_o3de_manifest_json_external_subdirectories : Adds the list of external_subdirectories
+#! in the user o3de_manifest.json to the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST property
+function(add_o3de_manifest_json_external_subdirectories)
+    o3de_get_manifest_path(manifest_path)
+    if(EXISTS ${manifest_path})
+        read_json_external_subdirs(o3de_manifest_external_subdirs ${manifest_path})
+        foreach(manifest_external_subdir ${o3de_manifest_external_subdirs})
+            file(REAL_PATH ${manifest_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER})
+
+            # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY
+            # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project
+            get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
+            if(NOT real_external_subdir IN_LIST current_external_subdirs)
+                set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir})
+                add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir})
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+#! Gather unique_list of all external subdirectories that is union
+#! of the engine.json, project.json, o3de_manifest.json and any gem.json files found visiting
+function(get_all_external_subdirectories output_subdirs)
+    get_property(all_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+    get_property(manifest_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST)
+    list(APPEND all_external_subdirs ${manifest_external_subdirs})
+    list(REMOVE_DUPLICATES all_external_subdirs)
+    set(${output_subdirs} ${all_external_subdirs} PARENT_SCOPE)
+endfunction()
+
+#! add_registered_gems_to_external_subdirs:
+#! Accepts a list of gem_names (which can be read from the project.json or engine.json)
+#! and cross checks them against union of all external subdirectories to determine the gem path.
+#! If that gem exist it is appended to LY_EXTERNAL_SUBDIRS so that that the build generator
+#! adds to the generated build project.
+#! Otherwise a fatal error is logged indicating that is not gem could not be found in the list of external subdirectories
+function(add_registered_gems_to_external_subdirs gem_names)
+    if (gem_names)
+        get_all_external_subdirectories(all_external_subdirs)
+        foreach(gem_name IN LISTS gem_names)
+            unset(gem_path)
+            o3de_find_gem(${gem_name} gem_path)
+            if (gem_path)
+                set_property(GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS ${gem_path} APPEND)
+            else()
+                list(JOIN all_external_subdirs "\n" external_subdirs_formatted)
+                message(SEND_ERROR "The gem \"${gem_name}\" from the \"gem_names\" field in the engine.json/project.json "
+                " could not be found in any gem.json from the following list of registered external subdirectories:\n"
+                "${external_subdirs_formatted}")
+                break()
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+function(add_subdirectory_on_external_subdirs)
+    # Lookup the paths of "gem_names" array all project.json files and engine.json
+    # and append them to the LY_EXTERNAL_SUBDIRS property
+    foreach(project ${LY_PROJECTS})
+        file(REAL_PATH ${project} full_directory_path BASE_DIRECTORY ${CMAKE_SOURCE_DIR})
+        o3de_read_json_array(gem_names ${full_directory_path}/project.json "gem_names")
+        add_registered_gems_to_external_subdirs("${gem_names}")
+    endforeach()
+    o3de_read_json_array(gem_names ${LY_ROOT_FOLDER}/engine.json "gem_names")
+    add_registered_gems_to_external_subdirs("${gem_names}")
+
+    get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS)
+    list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs})
+    list(REMOVE_DUPLICATES LY_EXTERNAL_SUBDIRS)
+    # Loop over the additional external subdirectories and invoke add_subdirectory on them
+    foreach(external_directory ${LY_EXTERNAL_SUBDIRS})
+        # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory
+        # This is to deal with potential situations where multiple external directories has the same last directory name
+        # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory
+        file(REAL_PATH ${external_directory} full_directory_path)
+        string(SHA256 full_directory_hash ${full_directory_path})
+        # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit
+        # when the external subdirectory contains relative paths of significant length
+        string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash)
+        # Use the last directory as the suffix path to use for the Binary Directory
+        cmake_path(GET external_directory FILENAME directory_name)
+        add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash})
+    endforeach()
+endfunction()

cmake/cmake_files.cmake

@@ -35,6 +35,7 @@ set(FILES
     Projects.cmake
     RuntimeDependencies.cmake
     SettingsRegistry.cmake
+    Subdirectories.cmake
     UnitTest.cmake
     Version.cmake
 )

engine.json

@@ -54,7 +54,7 @@
         "Gems/NvCloth",
         "Gems/PhysX",
         "Gems/PhysXDebug",
-        "Gems/Prefab",
+        "Gems/Prefab/PrefabBuilder",
         "Gems/Presence",
         "Gems/PrimitiveAssets",
         "Gems/Profiler",

scripts/o3de/o3de/cmake.py

@@ -150,7 +150,10 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
                 # If the in_gem_list was flipped to false, that means the currently parsed line contained the
                 # line end marker, so append that to the result_line
                 result_line += enable_gem_end_marker if not in_gem_list else ''
-                t_data.append(result_line + '\n')
+                # Strip of trailing whitespace. This also strips result lines which are empty of the indent
+                result_line = result_line.rstrip()
+                if result_line:
+                    t_data.append(result_line + '\n')
             else:
                 t_data.append(line)
 
@@ -165,11 +168,6 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
     return 0
 
 
-def get_project_gems(project_path: pathlib.Path,
-                     platform: str = 'Common') -> set:
-    return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform))
-
-
 def get_enabled_gems(cmake_file: pathlib.Path) -> set:
     """
     Gets a list of enabled gems from the cmake file
@@ -206,15 +204,6 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set:
     return gem_target_set
 
 
-def get_project_gem_paths(project_path:  pathlib.Path,
-                          platform: str = 'Common') -> set:
-    gem_names = get_project_gems(project_path, platform)
-    gem_paths = set()
-    for gem_name in gem_names:
-        gem_paths.add(manifest.get_registered(gem_name=gem_name, project_path=project_path))
-    return gem_paths
-
-
 def get_enabled_gem_cmake_file(project_name: str = None,
                                 project_path: str or pathlib.Path = None,
                                 platform: str = 'Common') -> pathlib.Path or None:

scripts/o3de/o3de/disable_gem.py

@@ -15,7 +15,7 @@ import os
 import pathlib
 import sys
 
-from o3de import cmake, manifest, utils
+from o3de import cmake, manifest, project_properties, utils
 
 logger = logging.getLogger('o3de.disable_gem')
 logging.basicConfig(format=utils.LOG_FORMAT)
@@ -68,8 +68,8 @@ def disable_gem_in_project(gem_name: str = None,
                      f' {project_path / "project.json"}, engine.json')
         return 1
     gem_path = pathlib.Path(gem_path).resolve()
-    # make sure this gem already exists if we're adding.  We can always remove a gem.
-    if not gem_path.exists():
+    # make sure the gem path is a directory
+    if not gem_path.is_dir():
         logger.error(f'Gem Path {gem_path} does not exist.')
         return 1
 
@@ -79,9 +79,6 @@ def disable_gem_in_project(gem_name: str = None,
         logger.error(f'Could not read gem.json content under {gem_path}.')
         return 1
 
-    # when removing we will try to do as much as possible even with failures so ret_val will be the last error code
-    ret_val = 0
-
     if not enabled_gem_file:
         enabled_gem_file = cmake.get_enabled_gem_cmake_file(project_path=project_path)
 
@@ -89,10 +86,14 @@ def disable_gem_in_project(gem_name: str = None,
     if not enabled_gem_file.is_file():
         logger.error(f'Enabled gem file {enabled_gem_file} is not present.')
         return 1
+
     # remove the gem
     error_code = cmake.remove_gem_dependency(enabled_gem_file, gem_json_data['gem_name'])
-    if error_code:
-        ret_val = error_code
+
+    # Remove the name of the gem from the project.json "gem_names" field if the gem is neither
+    # registered with the project.json nor engine.json
+    ret_val = project_properties.edit_project_props(project_path,
+                                                    delete_gem_names=gem_json_data['gem_name']) or error_code
 
     return ret_val
 

scripts/o3de/o3de/enable_gem.py

@@ -16,7 +16,7 @@ import os
 import pathlib
 import sys
 
-from o3de import cmake, manifest, register, validation, utils
+from o3de import cmake, manifest, project_properties, register, validation, utils
 
 logger = logging.getLogger('o3de.enable_gem')
 logging.basicConfig(format=utils.LOG_FORMAT)
@@ -33,7 +33,7 @@ def enable_gem_in_project(gem_name: str = None,
     :param gem_path: path to the gem to add
     :param project_name: name of to the project to add the gem to
     :param project_path: path to the project to add the gem to
-    :param enabled_gem_file_file: if this dependency goes/is in a specific file
+    :param enabled_gem_file: if this dependency goes/is in a specific file
     :return: 0 for success or non 0 failure code
     """
     # we need either a project name or path
@@ -80,8 +80,6 @@ def enable_gem_in_project(gem_name: str = None,
         logger.error(f'Could not read gem.json content under {gem_path}.')
         return 1
 
-
-    ret_val = 0
     if enabled_gem_file:
         # make sure this is a project has an enabled gems file
         if not enabled_gem_file.is_file():
@@ -96,17 +94,16 @@ def enable_gem_in_project(gem_name: str = None,
         if not project_enabled_gem_file.is_file():
             project_enabled_gem_file.touch()
 
-    # Before adding the gem_dependency check if the project is registered in either the project or engine
-    # manifest
+    # Before adding the gem_dependency check if the project is registered in either the project or engine manifest
     buildable_gems = manifest.get_engine_gems()
     buildable_gems.extend(manifest.get_project_gems(project_path))
-    # Convert each path to pathlib.Path object and filter out duplictes using dict.fromkeys
+    # Convert each path to pathlib.Path object and filter out duplicates using dict.fromkeys
     buildable_gems = list(dict.fromkeys(map(lambda gem_path_string: pathlib.Path(gem_path_string), buildable_gems)))
 
     ret_val = 0
-    # If the gem is not part of buildable set, it needs to be registered
-    if not gem_path in buildable_gems:
-        ret_val = register.register(gem_path=gem_path, external_subdir_project_path=project_path)
+    # If the gem is not part of buildable set, it's gem_name should be registered to the "gem_names" field
+    if gem_path not in buildable_gems:
+        ret_val = project_properties.edit_project_props(project_path, new_gem_names=gem_json_data['gem_name'])
 
     # add the gem if it is registered in either the project.json or engine.json
     ret_val = ret_val or cmake.add_gem_dependency(project_enabled_gem_file, gem_json_data['gem_name'])

scripts/o3de/o3de/engine_properties.py

@@ -18,11 +18,35 @@ from o3de import manifest, utils
 logger = logging.getLogger('o3de.engine_properties')
 logging.basicConfig(format=utils.LOG_FORMAT)
 
+def _edit_gem_names(engine_json: dict,
+                    new_gem_names: str or list = None,
+                    delete_gem_names: str or list = None,
+                    replace_gem_names: str or list = None):
+    if new_gem_names:
+        tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names
+        engine_json.setdefault('gem_names', []).extend(tag_list)
+    if delete_gem_names:
+        removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names
+        if 'gem_names' in engine_json:
+            for tag in removal_list:
+                if tag in engine_json['gem_names']:
+                    engine_json['gem_names'].remove(tag)
+    if replace_gem_names:
+        tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names
+        engine_json['gem_names'] = tag_list
+
+    # Remove duplicates from list
+    engine_json['gem_names'] = list(dict.fromkeys(engine_json.get('gem_names', [])))
+
 
 def edit_engine_props(engine_path: pathlib.Path = None,
                       engine_name: str = None,
                       new_name: str = None,
-                      new_version: str = None) -> int:
+                      new_version: str = None,
+                      new_gem_names: str or list = None,
+                      delete_gem_names: str or list = None,
+                      replace_gem_names: str or list = None
+                      ) -> int:
     if not engine_path and not engine_name:
         logger.error(f'Either a engine path or a engine name must be supplied to lookup engine.json')
         return 1
@@ -51,13 +75,20 @@ def edit_engine_props(engine_path: pathlib.Path = None,
     if new_version:
         engine_json_data['O3DEVersion'] = new_version
 
+    # Update the gem_names field in the engine.json
+    _edit_gem_names(engine_json_data, new_gem_names, delete_gem_names, replace_gem_names)
+
     return 0 if manifest.save_o3de_manifest(engine_json_data, pathlib.Path(engine_path) / 'engine.json') else 1
 
 def _edit_engine_props(args: argparse) -> int:
     return edit_engine_props(args.engine_path,
-                              args.engine_name,
-                              args.engine_new_name,
-                              args.engine_version)
+                             args.engine_name,
+                             args.engine_new_name,
+                             args.engine_version,
+                             args.add_gem_names,
+                             args.delete_gem_names,
+                             args.replace_gem_names
+                             )
 
 def add_parser_args(parser):
     group = parser.add_mutually_exclusive_group(required=True)
@@ -70,6 +101,13 @@ def add_parser_args(parser):
                        help='Sets the name for the engine.')
     group.add_argument('-ev', '--engine-version', type=str, required=False,
                        help='Sets the version for the engine.')
+    group = parser.add_mutually_exclusive_group(required=False)
+    group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False,
+                       help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)')
+    group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False,
+                       help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C')
+    group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False,
+                       help='Replace entirety of gem_names field with space delimited list of values')
     parser.set_defaults(func=_edit_engine_props)
 
 def add_args(subparsers) -> None:

scripts/o3de/o3de/project_properties.py

@@ -28,6 +28,27 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict:
     return proj_json
 
 
+def _edit_gem_names(proj_json: dict,
+                    new_gem_names: str or list = None,
+                    delete_gem_names: str or list = None,
+                    replace_gem_names: str or list = None):
+    if new_gem_names:
+        tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names
+        proj_json.setdefault('gem_names', []).extend(tag_list)
+    if delete_gem_names:
+        removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names
+        if 'gem_names' in proj_json:
+            for tag in removal_list:
+                if tag in proj_json['gem_names']:
+                    proj_json['gem_names'].remove(tag)
+    if replace_gem_names:
+        tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names
+        proj_json['gem_names'] = tag_list
+
+    # Remove duplicates from list
+    proj_json['gem_names'] = list(dict.fromkeys(proj_json.get('gem_names', [])))
+
+
 def edit_project_props(proj_path: pathlib.Path = None,
                        proj_name: str = None,
                        new_name: str = None,
@@ -38,7 +59,11 @@ def edit_project_props(proj_path: pathlib.Path = None,
                        new_icon: str = None,
                        new_tags: str or list = None,
                        delete_tags: str or list = None,
-                       replace_tags: str or list = None) -> int:
+                       replace_tags: str or list = None,
+                       new_gem_names: str or list = None,
+                       delete_gem_names: str or list = None,
+                       replace_gem_names: str or list = None
+                       ) -> int:
     proj_json = get_project_props(proj_name, proj_path)
     
     if not proj_json:
@@ -74,6 +99,8 @@ def edit_project_props(proj_path: pathlib.Path = None,
     if replace_tags:
         tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags
         proj_json['user_tags'] = tag_list
+    # Update the gem_names field in the project.json
+    _edit_gem_names(proj_json, new_gem_names, delete_gem_names, replace_gem_names)
 
     return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1
 
@@ -89,7 +116,10 @@ def _edit_project_props(args: argparse) -> int:
                               args.project_icon,
                               args.add_tags,
                               args.delete_tags,
-                              args.replace_tags)
+                              args.replace_tags,
+                              args.add_gem_names,
+                              args.delete_gem_names,
+                              args.replace_gem_names)
 
 
 def add_parser_args(parser):
@@ -118,6 +148,13 @@ def add_parser_args(parser):
                        help='Removes tag(s) from the user_tags property. Space delimited list (ex. -dt A B C')
     group.add_argument('-rt', '--replace-tags', type=str, nargs ='*', required=False,
                        help='Replace entirety of user_tags property with space delimited list of values')
+    group = parser.add_mutually_exclusive_group(required=False)
+    group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False,
+                       help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)')
+    group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False,
+                       help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C')
+    group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False,
+                       help='Replace entirety of gem_names field with space delimited list of values')
     parser.set_defaults(func=_edit_project_props)
 
 

scripts/o3de/tests/CMakeLists.txt

@@ -13,70 +13,77 @@ endif()
 # Add a test to test out the o3de package `o3de.py register` command
 ly_add_pytest(
     NAME o3de_register
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_register.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_register.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_cmake
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_cmake.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_cmake.py
+    TEST_SUITE smoke
+    EXCLUDE_TEST_RUN_TARGET_FROM_IDE
+)
+
+ly_add_pytest(
+    NAME o3de_disable_gem
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_disable_gem.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_enable_gem
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_enable_gem.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_enable_gem.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_global_project
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_global_project.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_global_project.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_manifest
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_manifest.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_manifest.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_engine_properties
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_properties.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_properties.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_project_properties
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_project_properties.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_project_properties.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_gem_properties
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_gem_properties.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_gem_properties.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_template
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_template.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_template.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )
 
 ly_add_pytest(
     NAME o3de_register_show
-    PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_print_registration.py
+    PATH ${CMAKE_CURRENT_LIST_DIR}/test_print_registration.py
     TEST_SUITE smoke
     EXCLUDE_TEST_RUN_TARGET_FROM_IDE
 )

scripts/o3de/tests/test_disable_gem.py

@@ -0,0 +1,200 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+import json
+
+import pytest
+import pathlib
+from unittest.mock import patch
+
+from o3de import cmake, disable_gem, enable_gem
+
+
+TEST_PROJECT_JSON_PAYLOAD = '''
+{
+    "project_name": "TestProject",
+    "origin": "The primary repo for TestProject goes here: i.e. http://www.mydomain.com",
+    "license": "What license TestProject uses goes here: i.e. https://opensource.org/licenses/MIT",
+    "display_name": "TestProject",
+    "summary": "A short description of TestProject.",
+    "canonical_tags": [
+        "Project"
+    ],
+    "user_tags": [
+        "TestProject"
+    ],
+    "icon_path": "preview.png",
+    "engine": "o3de-install",
+    "restricted_name": "projects",
+    "external_subdirectories": [
+    ]
+}
+'''
+
+TEST_GEM_JSON_PAYLOAD = '''
+{
+    "gem_name": "TestGem",
+    "display_name": "TestGem",
+    "license": "Apache-2.0 Or MIT",
+    "license_url": "https://github.com/o3de/o3de/blob/development/LICENSE.txt",
+    "origin": "Open 3D Engine - o3de.org",
+    "origin_url": "https://github.com/o3de/o3de",
+    "type": "Code",
+    "summary": "A short description of TestGem.",
+    "canonical_tags": [
+        "Gem"
+    ],
+    "user_tags": [
+        "TestGem"
+    ],
+    "icon_path": "preview.png",
+    "requirements": "Any requirement goes here.",
+    "documentation_url": "The link to the documentation goes here.",
+    "dependencies": [
+    ]
+}
+'''
+
+TEST_O3DE_MANIFEST_JSON_PAYLOAD = '''
+{
+    "o3de_manifest_name": "testuser",
+    "origin": "C:/Users/testuser/.o3de",
+    "default_engines_folder": "C:/Users/testuser/.o3de/Engines",
+    "default_projects_folder": "C:/Users/testuser/.o3de/Projects",
+    "default_gems_folder": "C:/Users/testuser/.o3de/Gems",
+    "default_templates_folder": "C:/Users/testuser/.o3de/Templates",
+    "default_restricted_folder": "C:/Users/testuser/.o3de/Restricted",
+    "default_third_party_folder": "C:/Users/testuser/.o3de/3rdParty",
+    "projects": [
+        "D:/MinimalProject"
+    ],
+    "external_subdirectories": [],
+    "templates": [],
+    "restricted": [],
+    "repos": [],
+    "engines": [
+        "D:/o3de/o3de"
+    ],
+    "engines_path": {
+        "o3de": "D:/o3de/o3de"
+    }
+}
+'''
+
[email protected](scope='class')
+def init_disable_gem_data(request):
+    class DisableGemData:
+        def __init__(self):
+            self.project_data = json.loads(TEST_PROJECT_JSON_PAYLOAD)
+            self.gem_data = json.loads(TEST_GEM_JSON_PAYLOAD)
+    request.cls.disable_gem = DisableGemData()
+
+
[email protected]('init_disable_gem_data')
+class TestDisableGemCommand:
+    @pytest.mark.parametrize("gem_path, project_path, gem_registered_with_project, gem_registered_with_engine,"
+                             "expected_result", [
+        pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0),
+        pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0),
+        pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0),
+        pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0),
+        ]
+    )
+    def test_disable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project,
+                                                             gem_registered_with_engine, expected_result):
+
+        project_gem_dependencies = []
+
+        def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path or None:
+            if project_name:
+                return project_path
+            elif gem_name:
+                return gem_path
+            return None
+
+        def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool:
+            if manifest_path == project_path / 'project.json':
+                self.disable_gem.project_data = new_project_data
+            return True
+
+        def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict or None:
+            if not manifest_path:
+                return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD)
+            return None
+
+        def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None):
+            return self.disable_gem.project_data
+
+        def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path):
+            return self.disable_gem.gem_data
+
+        def get_project_gems(project_path: pathlib.Path):
+            return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else []
+
+        def get_engine_gems():
+            return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else []
+
+        def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str):
+            project_gem_dependencies.append(gem_name)
+            return 0
+
+        def remove_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str):
+            project_gem_dependencies.remove(gem_name)
+            return 0
+
+        def get_enabled_gems(enable_gem_cmake_file: pathlib.Path) -> list:
+            return project_gem_dependencies
+
+
+        with patch('pathlib.Path.is_dir', return_value=True) as pathlib_is_dir_patch,\
+                patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_patch, \
+                patch('o3de.manifest.load_o3de_manifest', side_effect=load_o3de_manifest) as load_o3de_manifest_patch, \
+                patch('o3de.manifest.save_o3de_manifest', side_effect=save_o3de_manifest) as save_o3de_manifest_patch,\
+                patch('o3de.manifest.get_registered', side_effect=get_registered_path) as get_registered_patch,\
+                patch('o3de.manifest.get_gem_json_data', side_effect=get_gem_json_data) as get_gem_json_data_patch,\
+                patch('o3de.manifest.get_project_json_data', side_effect=get_project_json_data) as get_gem_json_data_patch,\
+                patch('o3de.manifest.get_project_gems', side_effect=get_project_gems) as get_project_gems_patch,\
+                patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\
+                patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch, \
+                patch('o3de.cmake.remove_gem_dependency',
+                      side_effect=remove_gem_dependency) as remove_gem_dependency_patch, \
+                patch('o3de.cmake.get_enabled_gems',
+                      side_effect=get_enabled_gems) as get_enabled_gems, \
+                patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch:
+
+            # Clear out any "gem_names" from the previous iterations
+            self.disable_gem.project_data.pop('gem_names', None)
+
+            # First enable the gem
+            assert enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path) == 0
+
+            # Check that the gem is enabled
+            gem_json = get_gem_json_data(gem_path, project_path)
+            project_json = get_project_json_data(project_path=project_path)
+            enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake")
+            assert gem_json.get('gem_name', '') in enabled_gems_list
+
+            # If the gem that is neither registered in the project.json nor engine.json,
+            # then it must appear in the "gem_names" field.
+            if not gem_registered_with_engine and not gem_registered_with_project:
+                assert gem_json.get('gem_name', '') in project_json.get('gem_names', [])
+            else:
+                assert gem_json.get('gem_name', '') not in project_json.get('gem_names', [])
+
+            # Now disable the gem
+            result = disable_gem.disable_gem_in_project(gem_path=gem_path, project_path=project_path)
+            assert result == expected_result
+
+            # Refresh the enabled_gems list and check for removal of the gem
+            gem_json = get_gem_json_data(gem_path, project_path)
+            project_json = get_project_json_data(project_path=project_path)
+            enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake")
+            assert gem_json.get('gem_name', '') not in enabled_gems_list
+
+            # If gem name should no longer appear in the "gem_names" field
+            assert gem_json.get('gem_name', '') not in project_json.get('gem_names', [])

scripts/o3de/tests/unit_test_enable_gem.py → scripts/o3de/tests/test_enable_gem.py

@@ -104,10 +104,11 @@ class TestEnableGemCommand:
         pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0),
         pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0),
         pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0),
+        pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0),
         ]
     )
-    def test_enable_gem_registers_gem_as_well(self, gem_path, project_path, gem_registered_with_project, gem_registered_with_engine,
-                        expected_result):
+    def test_enable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project,
+                                                             gem_registered_with_engine, expected_result):
 
         def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path:
             if project_name:
@@ -116,11 +117,8 @@ class TestEnableGemCommand:
                 return gem_path
             return None
 
-        def get_registered_gem_path(gem_name: str) -> pathlib.Path:
-            return gem_path
-
         def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool:
-            if manifest_path == project_path:
+            if manifest_path == project_path / 'project.json':
                 self.enable_gem.project_data = new_project_data
             return True
 
@@ -129,17 +127,17 @@ class TestEnableGemCommand:
                 return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD)
             return None
 
-        def get_project_json_data(project_path: pathlib.Path):
+        def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None):
             return self.enable_gem.project_data
 
         def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path):
             return self.enable_gem.gem_data
 
         def get_project_gems(project_path: pathlib.Path):
-            return [gem_path] if gem_registered_with_project else []
+            return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else []
 
         def get_engine_gems():
-            return [gem_path] if gem_registered_with_engine else []
+            return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else []
 
         def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str):
             return 0
@@ -155,11 +153,14 @@ class TestEnableGemCommand:
                 patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\
                 patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch,\
                 patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch:
+
+            self.enable_gem.project_data.pop('gem_names', None)
             result = enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path)
             assert result == expected_result
-            # If the gem isn't registered with the engine or project already it should now be registered with the project
-            if not gem_registered_with_engine and gem_registered_with_project:
-                # Prepend the project path to each external subdirectory
-                project_relative_subdirs = map(lambda subdir: (pathlib.Path(project_path) / subdir).as_posix(),
-                    self.enable_gem.project_data.get('external_subdirectories', []))
-                assert gem_path.as_posix() in project_relative_subdirs
+
+            gem_json = get_gem_json_data(gem_path, project_path)
+            project_json = get_project_json_data(project_path=project_path)
+            if not gem_registered_with_engine and not gem_registered_with_project:
+                assert gem_json.get('gem_name', '') in project_json.get('gem_names', [])
+            else:
+                assert gem_json.get('gem_name', '') not in project_json.get('gem_names', [])