Installer now supports tests and a new template adds test to existing projects (#18744)

* Adds the ability to make an installer which supports user tests

With these changes, the installer can be built with -DLY_DISABLE_TEST_MODULES=ON
but still generate an installer that users can use to run their OWN tests.

You can also build your own project based on the source code (not just the
installer) and use -DLY_DISABLE_TEST_MODULES=ON to disable the tests in the
engine, but leave your own tests enabled in your project.

This works on installer as well as source builds - LY_DISABLE_TEST_MODULES
disables tests in the engine.  They will only disable gem or project tests
if those gems or projects uses `if (Y_DISABLE_TEST_MODULES)` in their
own CMakeLists.txt, which they do not have to do, they can use their
own custom variable for example, to leave their tests on, but engine
tests off.

Tested on linux with installer and source builds, with a project
that has a googletest as well as a pytest (both work).

Both work with LY_DISABLE_TEST_MODULES=ON and OFF.
note that with LY_DISABLE_TEST_MODULES=ON, the engine tests are added
to your tests, if you're using a source build.  But with a installer build
the tests are not added to your tests since the installer is built with
them off and no project added.

* Adds a template for adding tests to any project
* Adds license snippet to tests
* Fixes android build

Signed-off-by: Nicholas Lawson <[email protected]>

CMakeLists.txt

@@ -129,8 +129,10 @@ endif()
 ly_delayed_generate_runtime_dependencies()
 
 # 6. Perform test impact framework post steps once all of the targets have been enumerated
-if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-    ly_test_impact_post_step()
+if(O3DE_TEST_IMPACT_NATIVE_TEST_TARGETS_ENABLED OR O3DE_TEST_IMPACT_PYTHON_TEST_TARGETS_ENABLED)
+    if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+        ly_test_impact_post_step()
+    endif()
 endif()
 
 # 7. Generate the O3DE find file and setup install locations for scripts, tools, assets etc., required by the engine

Code/Framework/AzTest/3rdParty/Findgoogletest.cmake

@@ -0,0 +1,155 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# GoogleTest and GoogleMock used to be two different repositories, but they are now combined into one repository.
+# This script fetches the combined repository and builds both libraries.
+
+if (TARGET 3rdParty::googletest::GTest)
+    return()
+endif()
+
+if (TARGET 3rdParty::googletest::GMock)
+    return()
+endif()
+
+# You should not be generating dependencies on googletest (via AzTest)
+# if you are on a platform that cannot actually compile googletest (See cmake/Platform/platformname/PAL_platformname.cmake)
+if (NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+    return()
+endif()
+
+set(GOOGLETEST_GIT_REPOSITORY "https://github.com/google/googletest.git")
+set(GOOGLETEST_GIT_TAG b514bdc898e2951020cbdca1304b75f5950d1f59) # tag name is v1.15.2
+set(GOOGLETEST_VERSION_STRING "v1.15.2")
+
+message(STATUS "AzTest uses googletest ${GOOGLETEST_VERSION_STRING} (BSD-3-Clause) from ${GOOGLETEST_GIT_REPOSITORY}")
+
+# there are optional flags you can pass FetchContent that make the project cleaner but require newer versions of cmake
+
+set(ADDITIONAL_FETCHCONTENT_FLAGS "")
+
+if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.25")
+    list(APPEND ADDITIONAL_FETCHCONTENT_FLAGS "SYSTEM")  # treat it as a 3p library, that is, do not issue warnings using the same warning level
+endif()
+
+set(CMAKE_POLICY_DEFAULT_CMP0148 OLD)
+set(OLD_CMAKE_WARN_DEPRECATED ${CMAKE_WARN_DEPRECATED})
+set(CMAKE_WARN_DEPRECATED FALSE CACHE BOOL "" FORCE)
+include(FetchContent)
+FetchContent_Declare(
+    googletest
+    GIT_REPOSITORY ${GOOGLETEST_GIT_REPOSITORY}
+    GIT_TAG ${GOOGLETEST_GIT_TAG}
+    ${ADDITIONAL_FETCHCONTENT_FLAGS}
+    GIT_SHALLOW TRUE
+)
+
+# CMAKE_ARGS don't actually affect FetchContent (currently by design) https://gitlab.kitware.com/cmake/cmake/-/issues/20799
+set(PYTHONINTERP_FOUND ON)
+set(PYTHON_EXECUTABLE ${Python_EXECUTABLE})
+set(PYTHON_VERSION_STRING ${Python_VERSION})
+set(gtest_force_shared_crt ON)
+set(BUILD_GMOCK ON)
+set(BUILD_GTEST ON)
+set(INSTALL_GTEST OFF)
+set(gmock_build_tests OFF)
+set(gtest_build_tests OFF)
+set(gtest_build_samples OFF)
+set(gtest_hide_internal_symbols ON)
+
+# Save values that apply globally that the 3rd party library may mess with
+# Some of these are null, hence the set(xxx "quoted value") so that if it isn't set it becomes the empty string.
+
+set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE)
+set(OLD_CMAKE_CXX_VISIBILITY_PRESET "${CMAKE_CXX_VISIBILITY_PRESET}")
+set(OLD_CMAKE_VISIBILITY_INLINES_HIDDEN "${CMAKE_VISIBILITY_INLINES_HIDDEN}")
+
+FetchContent_MakeAvailable(googletest)
+
+set(CMAKE_WARN_DEPRECATED ON CACHE BOOL "" FORCE)
+if (NOT "${OLD_CMAKE_CXX_VISIBILITY_PRESET}" STREQUAL "")
+    set(CMAKE_CXX_VISIBILITY_PRESET "${OLD_CMAKE_CXX_VISIBILITY_PRESET}")
+else()
+    unset(CMAKE_CXX_VISIBILITY_PRESET)
+endif()
+if (NOT "${OLD_CMAKE_VISIBILITY_INLINES_HIDDEN}" STREQUAL "")
+    set(CMAKE_VISIBILITY_INLINES_HIDDEN "${OLD_CMAKE_VISIBILITY_INLINES_HIDDEN}")
+else()
+    unset(CMAKE_VISIBILITY_INLINES_HIDDEN)
+endif()
+
+# Let's not clutter the root of any IDE folder structure with 3rd party dependencies
+# Setting the FOLDER makes it show up there in the solution build in VS and similarly
+# any other IDEs that organize in folders.
+set_target_properties(
+        gtest 
+        gmock 
+        gtest_main 
+        gmock_main 
+    PROPERTIES 
+        FOLDER "3rdParty Dependencies")
+
+unset(CMAKE_POLICY_DEFAULT_CMP0148)
+set(CMAKE_WARN_DEPRECATED ${OLD_CMAKE_WARN_DEPRECATED} CACHE BOOL "" FORCE)
+
+
+FetchContent_GetProperties(
+    googletest
+    SOURCE_DIR googletest_source_dir)
+    
+# GoogleTest and GoogleMock are considered public API - accessed by adding a dependency to AZ::AzTest.
+# This means we DO need to ship the include folders, in the installer and we do need the static libs, in case you link to AzTest.
+# By Default, no headers or includes are actually shipped, so we need to explicitly add them using ly_install.
+# doing so also helps us split them up between debug and non-debug versions
+
+# install the headers
+# note that source files include these headers using the name of the library, for example, they do
+# #include <gtest/gtest.h> and #include <gmock/gmock.h>, not just <gmock.h> or <gtest.h>
+# One option would be to just put these folders in (root)/include/gtest and (root)/include/gmock
+# and then add (root)/include as the include path for targets using these libraries.
+# However, that would cause any OTHER files in (root)/include to be visible to the compiler when all
+# you asked for is gmock/gtest.  So to avoid this, we instead copy the include files into
+# install/gmock_gtest/gmock 
+# install/gmock_gtest/gtest
+# and have install/gmock_gtest be the include path that is added to your compiler settings when you use these libraries, 
+# so that #include <gtest/gtest> still works (since its relative to that path) BUT it adds no additional other
+# files or libraries to your include path that would otherwise leak in.
+ly_install(DIRECTORY ${googletest_source_dir}/googletest/include/gtest DESTINATION include/gmock_gtest COMPONENT CORE)
+ly_install(DIRECTORY ${googletest_source_dir}/googlemock/include/gmock DESTINATION include/gmock_gtest COMPONENT CORE)
+# include the license files just for good measure (Although using the library will disclose the
+# source git repository and version anyway.
+ly_install(FILES ${googletest_source_dir}/LICENSE COMPONENT CORE DESTINATION include/gmock_gtest/gtest)
+ly_install(FILES ${googletest_source_dir}/LICENSE COMPONENT CORE DESTINATION include/gmock_gtest/gmock)
+
+# Make the installer find-files be used when in an installer pre-built mode
+ly_install(FILES ${CMAKE_CURRENT_LIST_DIR}/Installer/Findgoogletest.cmake DESTINATION cmake/3rdParty)
+
+# install the libraries making sure to use different directories for debug/release/etc
+set(BASE_LIBRARY_FOLDER "lib/${PAL_PLATFORM_NAME}")
+foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES)
+    string(TOUPPER ${conf} UCONF)
+    ly_install(TARGETS gtest gmock
+        ARCHIVE
+            DESTINATION "${BASE_LIBRARY_FOLDER}/${conf}"
+            COMPONENT ${LY_INSTALL_PERMUTATION_COMPONENT}_${UCONF}
+            CONFIGURATIONS ${conf}
+    )
+endforeach()
+
+# do not use ly_create_alias as that will autogenerate duplicate calls inside the installer.
+# ly_create_alias would create "GTest", "GMock" and "GTest::GTest" and "GMock::GMock" aliases, as well
+# as add them to an internal list of targets to autogenerate imported targets for.
+
+# That's okay if you're going to let the installer do its own thing, but in our case, we're including 
+# a FindXXXX in the installer, so create them directly, without calling the LY macro
+add_library(3rdParty::googletest::GTest ALIAS gtest)
+add_library(3rdParty::googletest::GMock ALIAS gmock)
+
+set(googletest_FOUND TRUE)
+
+

Code/Framework/AzTest/3rdParty/Installer/Findgoogletest.cmake

@@ -0,0 +1,45 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+if (TARGET 3rdParty::googletest::GTest)
+    return()
+endif()
+
+if (TARGET 3rdParty::googletest::GMock)
+    return()
+endif()
+
+# You should not be generating dependencies on googletest (via AzTest)
+# if you are on a platform that cannot actually compile googletest (See cmake/Platform/platformname/PAL_platformname.cmake)
+if (NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+    return()
+endif()
+
+
+# This file is run when in pre-built installer mode, and thus GTest and GMock are pre-built static libraries
+# ALWAYS DISCLOSE THE USE OF 3rd Party Libraries.  Even if they are internally linked.
+message(STATUS "AzTest library uses googletest v1.15.2 (BSD-3-Clause) from https://github.com/google/googletest.git")
+
+set(BASE_LIBRARY_FOLDER "${LY_ROOT_FOLDER}/lib/${PAL_PLATFORM_NAME}")
+
+add_library(gtest STATIC IMPORTED GLOBAL)
+set_target_properties(gtest PROPERTIES 
+    IMPORTED_LOCATION       "${BASE_LIBRARY_FOLDER}/profile/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    IMPORTED_LOCATION_DEBUG "${BASE_LIBRARY_FOLDER}/debug/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ly_target_include_system_directories(TARGET gtest INTERFACE "${LY_ROOT_FOLDER}/include/gmock_gtest")
+
+add_library(gmock STATIC IMPORTED GLOBAL)
+set_target_properties(gmock PROPERTIES 
+    IMPORTED_LOCATION       "${BASE_LIBRARY_FOLDER}/profile/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    IMPORTED_LOCATION_DEBUG "${BASE_LIBRARY_FOLDER}/debug/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ly_target_include_system_directories(TARGET gmock INTERFACE "${LY_ROOT_FOLDER}/include/gmock_gtest")
+    
+add_library(3rdParty::googletest::GTest ALIAS gtest)
+add_library(3rdParty::googletest::GMock ALIAS gmock)
+
+set(googletest_FOUND TRUE)

Code/Framework/AzTest/CMakeLists.txt

@@ -4,61 +4,19 @@
 #
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 #
-#
-
-if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-    set(GOOGLETEST_GIT_REPOSITORY "https://github.com/google/googletest.git")
-    set(GOOGLETEST_GIT_TAG b514bdc898e2951020cbdca1304b75f5950d1f59) # tag name is v1.15.2
-    set(GOOGLETEST_VERSION_STRING "v1.15.2")
-
-    # there are optional flags you can pass FetchContent that make the project cleaner but require newer versions of cmake
-    
-    set(ADDITIONAL_FETCHCONTENT_FLAGS "")
 
-    if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.25")
-    list(APPEND ADDITIONAL_FETCHCONTENT_FLAGS "SYSTEM")  # treat it as a 3p library, that is, do not issue warnings using the same warning level
-    endif()
-
-    o3de_pal_dir(pal_aztest_dir ${CMAKE_CURRENT_LIST_DIR}/AzTest/Platform/${PAL_PLATFORM_NAME} ${O3DE_ENGINE_RESTRICTED_PATH} ${LY_ROOT_FOLDER})
-
-    set(CMAKE_POLICY_DEFAULT_CMP0148 OLD)
-    set(OLD_CMAKE_WARN_DEPRECATED ${CMAKE_WARN_DEPRECATED})
-    set(CMAKE_WARN_DEPRECATED FALSE CACHE BOOL "" FORCE)
-    include(FetchContent)
-    message(STATUS "Using googletest from ${GOOGLETEST_GIT_REPOSITORY} ${GOOGLETEST_VERSION_STRING}")
-    FetchContent_Declare(
-        googletest
-        GIT_REPOSITORY ${GOOGLETEST_GIT_REPOSITORY}
-        GIT_TAG ${GOOGLETEST_GIT_TAG}
-        ${ADDITIONAL_FETCHCONTENT_FLAGS}
-        CMAKE_ARGS "-DCMAKE_WARN_DEPRECATED=FALSE;-Dgtest_force_shared_crt=ON;-DBUILD_GMOCK=ON;-DBUILD_GTEST=ON;-DINSTALL_GTEST=OFF;-Dgmock_build_tests=OFF;-Dgtest_build_tests=OFF;-Dgtest_build_samples=OFF;-Dgtest_hide_internal_symbols=ON"
-    )
-    
-    set(PYTHONINTERP_FOUND ON)
-    set(PYTHON_EXECUTABLE ${Python_EXECUTABLE})
-    set(PYTHON_VERSION_STRING ${Python_VERSION})
+# Note we do not use if(PAL_TRAIT_BUILD_TESTS_SUPPORTED) on this file.
+# This is because that flag is used to determine if the tests themselves should be built or not
+# But this is how you would build your own tests, and must be included in the installer, even if
+# we don't include the tests in the installer.
 
-    FetchContent_MakeAvailable(googletest)
+# when compiling from source, use the 3rdParty/Find***.cmake files
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/3rdParty)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
 
-    # Let's not clutter the root of any IDE folder structure with 3rd party dependencies
-    # Setting the FOLDER makes it show up there in the solution build in VS and similarly
-    # any other IDEs that organize in folders.
-    # Setting EXCLUDE_FROM_ALL Makes it not appear in the first place.
-    set_target_properties(
-            gtest 
-            gmock 
-            gtest_main 
-            gmock_main 
-        PROPERTIES 
-            FOLDER "3rdParty Dependencies" 
-            EXCLUDE_FROM_ALL TRUE)
-
-    unset(CMAKE_POLICY_DEFAULT_CMP0148)
-    set(CMAKE_WARN_DEPRECATED ${OLD_CMAKE_WARN_DEPRECATED} CACHE BOOL "" FORCE)
-
-    ly_create_alias(NAME googletest::GTest NAMESPACE 3rdParty TARGETS gtest)
-    ly_create_alias(NAME googletest::GMock NAMESPACE 3rdParty TARGETS gmock)
+o3de_pal_dir(pal_aztest_dir ${CMAKE_CURRENT_LIST_DIR}/AzTest/Platform/${PAL_PLATFORM_NAME} ${O3DE_ENGINE_RESTRICTED_PATH} ${LY_ROOT_FOLDER})
 
+if (PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
     ly_add_target(
         NAME AzTest STATIC
         NAMESPACE AZ
@@ -71,34 +29,11 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
                 ${pal_aztest_dir}
         BUILD_DEPENDENCIES
             PUBLIC
-                3rdParty::googletest::GMock
+                3rdParty::googletest::GMock   # This dependency is what causes the find files to run.
                 3rdParty::googletest::GTest
                 3rdParty::GoogleBenchmark
                 AZ::AzCore
         PLATFORM_INCLUDE_FILES
             ${pal_aztest_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
     )
-
-    # Maintainers, please note that when building O3DE from source, this entire script file you are looking at right now
-    # will execute during configure.  However, only the below literal block between the [[ and ]] below will
-    # be included in the built installer, the rest of this file will not be, the above targets will be synthesized
-    # as if they were prebuilt libraries (For example, the AzTest library declared above will instead be autogenerated in its
-    # own file as if it were a STATIC IMPORTED GLOBAL target instead of a built-from-source-code target).
-    # This means that the below block must declare all necessary 3rd Party libraries fetched and built by fetchcontent,
-    # and cannot refer to variables from the rest of this script file - only the snippet below will exist, and must be self-contained.
-
-    # Export external googletest targets for installers.
-    set_property(DIRECTORY APPEND PROPERTY O3DE_SUBDIRECTORY_INSTALL_CODE [[
-        add_library(gtest STATIC IMPORTED GLOBAL)
-        set_target_properties(gtest PROPERTIES IMPORTED_LOCATION "${LY_ROOT_FOLDER}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
-        ly_target_include_system_directories(TARGET gtest INTERFACE "${LY_ROOT_FOLDER}/include/gtest")
-        
-        add_library(gmock STATIC IMPORTED GLOBAL)
-        set_target_properties(gmock PROPERTIES IMPORTED_LOCATION "${LY_ROOT_FOLDER}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}")
-        ly_target_include_system_directories(TARGET gmock INTERFACE "${LY_ROOT_FOLDER}/include/gmock")
-            
-        ly_create_alias(NAME googletest::GTest NAMESPACE 3rdParty TARGETS gtest)
-        ly_create_alias(NAME googletest::GMock NAMESPACE 3rdParty TARGETS gmock)
-    ]]
-    )
 endif()

Code/Tools/AzTestRunner/CMakeLists.txt

@@ -6,12 +6,21 @@
 #
 #
 
+# if this platform does not support google tests, it cannot support testrunner and there is no point in continuing.
+if(NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+    return()
+endif()
+
 o3de_pal_dir(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME} ${O3DE_ENGINE_RESTRICTED_PATH} ${LY_ROOT_FOLDER})
 
 include(${pal_dir}/platform_traits_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
 
-if(PAL_TRAIT_AZTESTRUNNER_SUPPORTED AND PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-    
+# note that LY_DISABLE_TEST_MODULES is a CMake variable that controls whether
+# test modules are built or not and it should be interpreted as "build our own
+# tests"... not as "support no testing at all if set".
+# We still need to build and ship AzTestRunner as part of the engine and installer
+# so that people using the installer can run their OWN tests.
+if(PAL_TRAIT_AZTESTRUNNER_SUPPORTED)
     ly_add_target(
         NAME AzTestRunner ${PAL_TRAIT_AZTESTRUNNER_LAUNCHER_TYPE}
         NAMESPACE AZ
@@ -43,7 +52,6 @@ if(PAL_TRAIT_AZTESTRUNNER_SUPPORTED AND PAL_TRAIT_BUILD_TESTS_SUPPORTED)
 
         ly_add_googletest(
             NAME AZ::AzTestRunner.Tests
-            LABELS REQUIRES_tiaf
         )
 
     endif()

Code/Tools/TestImpactFramework/Platform/Windows/PAL_windows.cmake

@@ -6,4 +6,4 @@
 #
 #
 
-set(PAL_TRAIT_TEST_IMPACT_FRAMEWORK_SUPPORTED TRUE)
+set(PAL_TRAIT_TEST_IMPACT_FRAMEWORK_SUPPORTED FALSE)

Gems/TestAssetBuilder/Code/CMakeLists.txt

@@ -6,7 +6,8 @@
 #
 #
 
-if(NOT PAL_TRAIT_BUILD_HOST_TOOLS)
+# This gem itself is a test module.
+if(NOT PAL_TRAIT_BUILD_HOST_TOOLS OR NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
     return()
 endif()
 

Templates/AddTestsToProject/Template/Gem/Tests/custom_google_test.cmake

@@ -0,0 +1,37 @@
+# {BEGIN_LICENSE}
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# {END_LICENSE}
+
+# Include this file in your Project/Gem/CMakeLists.txt using "include(Tests/custom_google_test.cmake)" at the end.
+
+# Add a cache variable or set -DDISABLE_GEM_TESTS=ON to disable tests for this gem
+# PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED is defined in the O3DE CMake code and will be
+# FALSE if the current platform (like IOS for example) does not support Google Test
+if (NOT DISABLE_GEM_TESTS AND PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+    # add the test cpp code (add new files in the _test_files.cmake file)
+    ly_add_target(
+        NAME ${gem_name}.Tests ${PAL_TRAIT_TEST_TARGET_TYPE}
+        NAMESPACE Gem
+        FILES_CMAKE
+            ${CMAKE_CURRENT_LIST_DIR}/test_files.cmake
+        BUILD_DEPENDENCIES
+            PRIVATE
+                AZ::AzTest
+    )
+
+    # Register the test so that it runs when you run ctest
+    # eg, ctest -P <build-config> --test-dir <path-to-your-build-dir>
+    # eg, ctest -P profile --test-dir build/windows
+    # eg, ctest -P debug --test-dir build\linux
+    ly_add_googletest(
+            NAME Gem::${gem_name}.Tests
+        # optional parameters: 
+        # TEST_SUITE "main" or "sandbox" or "smoke"
+        # TARGET name of ly_target to run, if it does not match NAME of this test
+    )
+endif()
+

Templates/AddTestsToProject/Template/Gem/Tests/custom_pytest.cmake

@@ -0,0 +1,29 @@
+# {BEGIN_LICENSE}
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# {END_LICENSE}
+
+# Include this file in your Project/Gem/CMakeLists.txt using "include(Tests/custom_pytest.cmake)" at the end.
+
+# Add a cache variable or set -DDISABLE_GEM_TESTS=ON to disable tests for this gem
+# PAL_TRAIT_TEST_PYTEST_SUPPORTED is defined in the O3DE CMake code and will be
+# FALSE if the current platform (like IOS for example) does not support python tests
+if (DISABLE_GEM_TESTS OR NOT PAL_TRAIT_TEST_PYTEST_SUPPORTED)
+    return()
+endif()
+
+# Register the test so that it runs when you run ctest
+# eg, ctest -P <build-config> --test-dir <path-to-your-build-dir-you-used-when-you-ran-cmake>
+# eg, ctest -P profile --test-dir build/windows
+# eg, ctest -P debug --test-dir build\linux
+# Unit tests.
+ly_add_pytest(
+    NAME PyTestMain_main_no_gpu  
+        # the name of the test also has extra optional tags on the end of it, eg
+        # optional addition of "_main", "_sandbox", "_periodic", "_smoke" puts it in that suite.  Unspecified = main
+        # "_no_gpu" / "_requires_gpu" = it does not require a gpu to run, or it does.   Used to filter tests out on headless. 
+    PATH ${CMAKE_CURRENT_LIST_DIR} # this can be a specific .pyfile or a directory to be scanned for files matching "*test*.py"
+)

Templates/AddTestsToProject/Template/Gem/Tests/test_example_pytest.py

@@ -0,0 +1,23 @@
+# {BEGIN_LICENSE}
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# {END_LICENSE}
+
+import unittest.mock as mock
+import pytest
+
+# This line is here to prove that you can import things from ly_test_tools or other parts
+# of the o3de toolkit to help you make tests.  See the documentation as well as existing tests
+# in the gems and in the o3de repository (especially AutomatedTesting project) for examples on how
+# to run things inside the editor, etc.
+import ly_remote_console.remote_console_commands as remote_console
+
+# The rest is standard pytest functionality, you can write tests here as you would in pytest.
[email protected]
+class TestExample():
+    def test_ExampleTest(self):
+        assert True
+

Templates/AddTestsToProject/Template/Gem/Tests/test_files.cmake

@@ -0,0 +1,11 @@
+# {BEGIN_LICENSE}
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+# {END_LICENSE}
+
+set(FILES
+    testmain.cpp # Relative to the current folder.
+)

Templates/AddTestsToProject/Template/Gem/Tests/testmain.cpp

@@ -0,0 +1,21 @@
+// {BEGIN_LICENSE}
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+// {END_LICENSE}
+
+#include <AzTest/AzTest.h>
+
+ TEST(MyTestSuiteName, ExampleTest)
+ {
+     ASSERT_TRUE(true);
+ }
+
+ // If you need to have your own environment preconditions
+ // you can replace the test env with your own.
+ AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV);
+ 

Templates/AddTestsToProject/preview.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:248e3ffe1fc9ffc02afb2ba8914e222a5a5d13ac45a48b98c95ee062e959a94c
+size 4475

Templates/AddTestsToProject/template.json

@@ -0,0 +1,42 @@
+{
+    "template_name": "AddTestsToProject",
+    "origin": "https://github.com/o3de/o3de",
+    "license": "Apache-2.0",
+    "display_name": "Add Tests to your Project",
+    "summary": "Adds tests to an existing project.  Point it at your project's root, then open YourProject/Gem/Tests folder for more info",
+    "canonical_tags": [],
+    "user_tags": [
+        "AddTestsToProject"
+    ],
+    "icon_path": "preview.png",
+    "copyFiles": [
+        {
+            "file": "Gem/Tests/custom_google_test.cmake",
+            "isTemplated": false
+        },
+        {
+            "file": "Gem/Tests/custom_pytest.cmake",
+            "isTemplated": false
+        },
+        {
+            "file": "Gem/Tests/test_example_pytest.py",
+            "isTemplated": false
+        },
+        {
+            "file": "Gem/Tests/test_files.cmake",
+            "isTemplated": false
+        },
+        {
+            "file": "Gem/Tests/testmain.cpp",
+            "isTemplated": false
+        }
+    ],
+    "createDirectories": [
+        {
+            "dir": "Gem"
+        },
+        {
+            "dir": "Gem/Tests"
+        }
+    ]
+}

Templates/CMakeLists.txt

@@ -9,6 +9,7 @@
 ly_install_directory(
     DIRECTORIES
         AssetGem
+        AddTestsToProject
         CppToolGem
         DefaultGem
         PrebuiltGem

Tools/LyTestTools/tests/CMakeLists.txt

@@ -8,6 +8,10 @@
 # LyTestTools tests.
 #
 
+if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    return()
+endif()
+
 # Unit tests.
 ly_add_pytest(
     NAME LyTestTools_Unit

Tools/RemoteConsole/ly_remote_console/tests/CMakeLists.txt

@@ -8,6 +8,10 @@
 # Ly Remote Console tests.
 #
 
+if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    return()
+endif()
+
 # Unit tests.
 ly_add_pytest(
     NAME RemoteConsole_UnitTests_main_no_gpu

cmake/3rdParty/script-only-mode/PostProcessScriptOnlyMappings.cmake

@@ -85,7 +85,8 @@ foreach(data_file IN LISTS file_list)
         endif()
     endif()
 
-    string(APPEND final_data "add_library(${target_name} INTERFACE IMPORTED GLOBAL)\n")
+    string(APPEND final_data "if (NOT TARGET ${target_name})\n")
+    string(APPEND final_data "    add_library(${target_name} INTERFACE IMPORTED GLOBAL)\n")
 
     # binary_folder can be a genex but will always be the Default folder since monolithic
     # cannot be combined with script-only
@@ -99,12 +100,13 @@ foreach(data_file IN LISTS file_list)
                 set(file_name "${installer_binaries}/${file_name}")
             endif()
             string(APPEND final_data 
-                    "ly_add_target_files(TARGETS ${target_name}\n\
+                    "    ly_add_target_files(TARGETS ${target_name}\n\
                         FILES\n\
                             \"${file_name}\"\n\
                         OUTPUT_SUBDIRECTORY \"${relative_path}\")\n\n")
         endforeach()        
     endif()
+    string(APPEND final_data "endif() # NOT TARGET ${target_name}\n")
 
     string(REPLACE "::" "__" CLEAN_TARGET_NAME "${target_name}")  
     file(WRITE "${output_directory}/${CLEAN_TARGET_NAME}.cmake" "${final_data}")

cmake/LYPython.cmake

@@ -337,9 +337,10 @@ if (NOT CMAKE_SCRIPT_MODE_FILE)
 
         # we also need to make sure any custom packages are installed.
         # this costs a moment of time though, so we'll only do it based on stamp files.
+        ly_pip_install_local_package_editable(${LY_ROOT_FOLDER}/Tools/LyTestTools ly-test-tools)
+        ly_pip_install_local_package_editable(${LY_ROOT_FOLDER}/Tools/RemoteConsole/ly_remote_console ly-remote-console)
         if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND NOT INSTALLED_ENGINE)
-            ly_pip_install_local_package_editable(${LY_ROOT_FOLDER}/Tools/LyTestTools ly-test-tools)
-            ly_pip_install_local_package_editable(${LY_ROOT_FOLDER}/Tools/RemoteConsole/ly_remote_console ly-remote-console)
+            # These are test tools private to the AutomatedTesting project itself, which is not relevant to the installer.
             ly_pip_install_local_package_editable(${LY_ROOT_FOLDER}/AutomatedTesting/Gem/PythonTests/EditorPythonTestTools editor-python-test-tools)
         endif()
 

cmake/LYTestWrappers.cmake

@@ -95,10 +95,6 @@ endfunction()
 #      ly_add_* function below, not by user code
 # sets LY_ADDED_TEST_NAME to the fully qualified name of the test, in parent scope
 function(ly_add_test)
-    if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-        return()
-    endif()
-
     set(options EXCLUDE_TEST_RUN_TARGET_FROM_IDE)
     set(one_value_args NAME PARENT_NAME TEST_LIBRARY TEST_SUITE TIMEOUT)
     set(multi_value_args TEST_REQUIRES TEST_COMMAND NON_IDE_PARAMS RUNTIME_DEPENDENCIES COMPONENT LABELS)
@@ -366,8 +362,8 @@ endfunction()
 # \arg:TIMEOUT (optional) The timeout in seconds for the module. If not set, will have its timeout set by ly_add_test to the default timeout.
 # \arg:EXCLUDE_TEST_RUN_TARGET_FROM_IDE(bool) - If set the test run target will be not be shown in the IDE
 function(ly_add_googletest)
-    if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-        message(FATAL_ERROR "Platform does not support test targets")
+    if(NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+        message(FATAL_ERROR "Googletest unavailable on this platform.  check PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED before using ly_add_googletest")
     endif()
 
     set(one_value_args NAME TARGET TEST_SUITE)
@@ -385,11 +381,6 @@ function(ly_add_googletest)
     # google test suites are supported
     set_property(GLOBAL APPEND PROPERTY LY_AZTESTRUNNER_TEST_MODULES "${target_name}")
 
-    if(NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
-        return()
-    endif()
-
-
     if (ly_add_googletest_TEST_SUITE AND NOT ly_add_googletest_TEST_SUITE STREQUAL "main")
         # if a suite is specified, we filter to only accept things which match that suite (in c++)
         set(non_ide_params "--gtest_filter=*SUITE_${ly_add_googletest_TEST_SUITE}*")
@@ -460,10 +451,11 @@ endfunction()
 #      NOTE: Not used if a custom TEST_COMMAND is supplied
 # \arg:TIMEOUT (optional) The timeout in seconds for the module. If not set, will have its timeout set by ly_add_test to the default timeout.
 function(ly_add_googlebenchmark)
-    if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-        message(FATAL_ERROR "Platform does not support test targets")
-    endif()
     if(NOT PAL_TRAIT_TEST_GOOGLE_BENCHMARK_SUPPORTED)
+        # Special case for benchmarks.  Most of the time, the benchmark is inside an existing module
+        # that also contains non-benchmark googletest tests.  In this case, its easier to just early out and ignore
+        # the request to add a benchmark rather than to FATAL ERROR which would require a separate if check for every
+        # module that contains benchmarks.
         return()
     endif()
 

cmake/PAL.cmake

@@ -438,6 +438,8 @@ include(${pal_cmake_dir}/Toolchain_${PAL_PLATFORM_NAME_LOWERCASE}.cmake OPTIONAL
 
 set(LY_DISABLE_TEST_MODULES FALSE CACHE BOOL "Option to forcibly disable the inclusion of test targets in the build")
 
-if(LY_DISABLE_TEST_MODULES)
+if(LY_DISABLE_TEST_MODULES OR NOT PAL_TRAIT_TEST_GOOGLE_TEST_SUPPORTED)
+    # AzTest library, which requires google test, is not supported on this platform
+    # so none of the tests can build either.
     ly_set(PAL_TRAIT_BUILD_TESTS_SUPPORTED FALSE)
 endif()

cmake/Platform/iOS/RuntimeDependencies_ios.cmake

@@ -103,21 +103,20 @@ function(ly_delayed_generate_runtime_dependencies)
         endif()
     endforeach()
 
-    if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-        add_dependencies("AzTestRunner" ${test_runner_dependencies})
-        
-        # We still need to add indirect dependencies(eg. 3rdParty)
-        unset(dependencies)
-        o3de_get_filtered_dependencies_for_target(dependencies AzTestRunner)
-
-        if(dependencies)
-            set_target_properties("AzTestRunner"
-                PROPERTIES
-                XCODE_EMBED_FRAMEWORKS "${dependencies}"
-                XCODE_EMBED_FRAMEWORKS_CODE_SIGN_ON_COPY TRUE
-                XCODE_ATTRIBUTE_LD_RUNPATH_SEARCH_PATHS "@executable_path/Frameworks"
-            )
-        endif()
+
+    add_dependencies("AzTestRunner" ${test_runner_dependencies})
+    
+    # We still need to add indirect dependencies(eg. 3rdParty)
+    unset(dependencies)
+    o3de_get_filtered_dependencies_for_target(dependencies AzTestRunner)
+
+    if(dependencies)
+        set_target_properties("AzTestRunner"
+            PROPERTIES
+            XCODE_EMBED_FRAMEWORKS "${dependencies}"
+            XCODE_EMBED_FRAMEWORKS_CODE_SIGN_ON_COPY TRUE
+            XCODE_ATTRIBUTE_LD_RUNPATH_SEARCH_PATHS "@executable_path/Frameworks"
+        )
     endif()
 
 endfunction()

cmake/TestImpactFramework/TestImpactTestTargetConfig.cmake

@@ -5,10 +5,6 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 #
 #
-if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
-    return()
-endif()
-
 # Path to test instrumentation binary
 set(O3DE_TEST_IMPACT_INSTRUMENTATION_BIN "" CACHE PATH "Path to test impact framework instrumentation binary")
 

cmake/install/Findo3de.cmake.in

@@ -34,15 +34,6 @@ o3de_current_file_path(find_o3de_path)
 cmake_path(SET engine_root_folder NORMALIZE ${find_o3de_path}/..)
 set_property(GLOBAL PROPERTY O3DE_ENGINE_ROOT_FOLDER "${engine_root_folder}")
 
-# If the generation of the SDK layout had LY_DISABLE_TEST_MODULES cache variable set, 
-# then the PAL_TRAIT_BUILD_TESTS_SUPPORTED value needs to be propagated as FALSE
-set(disable_test_modules @LY_DISABLE_TEST_MODULES@)
-if (disable_test_modules)
-    set_property(GLOBAL PROPERTY PAL_TRAIT_BUILD_TESTS_SUPPORTED_DEFAULT FALSE)
-else()
-    set_property(GLOBAL PROPERTY PAL_TRAIT_BUILD_TESTS_SUPPORTED_DEFAULT TRUE)
-endif()
-
 if ($ENV{O3DE_SNAP})
     list(APPEND CMAKE_REQUIRED_INCLUDES "$ENV{SNAP}/usr/include;$ENV{SNAP}/usr/include/x86_64-linux-gnu")
     list(APPEND CMAKE_REQUIRED_LINK_OPTIONS "Wl,-L$ENV{SNAP}/usr/lib/x86_64-linux-gnu")
@@ -58,6 +49,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${engine_root_folder}cmake/CompilerSettings.cm
 macro(o3de_initialize)
     set(INSTALLED_ENGINE TRUE)
     set(LY_PROJECTS ${CMAKE_SOURCE_DIR})
+    include(CTest) # automatically triggers enable_testing
     o3de_current_file_path(current_path)
     add_subdirectory(${current_path}/.. o3de)
 endmacro()

engine.json

@@ -113,6 +113,7 @@
     ],
     "templates": [
         "Templates/AssetGem",
+        "Templates/AddTestsToProject",
         "Templates/CppToolGem",
         "Templates/DefaultComponent",
         "Templates/DefaultGem",

scripts/commit_validation/CMakeLists.txt

@@ -5,6 +5,9 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 #
 #
+if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    return()
+endif()
 
 ly_add_pytest(
     NAME test_commit_validation

scripts/detect_file_changes/CMakeLists.txt

@@ -6,6 +6,10 @@
 #
 #
 
+if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    return()
+endif()
+
 ly_add_pytest(
     NAME test_detect_file_changes
     PATH ${CMAKE_CURRENT_LIST_DIR}