Add manual test for rga (#14385)

What does this PR do?

Add a manual test for radeon gpu analyzer, the test is using test_suite periodic, so it won't be run by automated test.
The test will copy the test data into the Assets folder, and check if the analysis data had been generated.
How was this PR tested?

Test locally with command >python\python.cmd -m pytest --build-directory .\build\windows\bin\profile .\AutomatedTesting\Gem\PythonTests\Atom\TestSuite_Manual.py

Note the test uses many functions from hydra_ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges.py, so I modulize the functions and put them in atom_tools_utils.py.

The chain dependency test still worked correctly when I run command >python\python.cmd -m pytest --build-directory .\build\windows\bin\profile .\AutomatedTesting\Gem\PythonTests\Atom\TestSuite_Periodic_Null_Render_01.py

---------

Signed-off-by: Ethan Chen <[email protected]>

AutomatedTesting/Gem/PythonTests/Atom/CMakeLists.txt

@@ -171,4 +171,16 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED)
         COMPONENT
             Atom
     )
+    ly_add_pytest(
+        NAME AutomatedTesting::RGATest 
+        TEST_SUITE sandbox
+        TIMEOUT 1200
+        PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Manual.py
+        TEST_SERIAL 
+        RUNTIME_DEPENDENCIES 
+            AssetProcessor
+            AutomatedTesting.Assets
+        COMPONENT
+            Atom
+    )
 endif()

AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Manual.py

@@ -0,0 +1,15 @@
+"""
+Copyright (c) Contributors to the Open 3D Engine Project.
+For complete copyright and license terms please see the LICENSE at the root of this distribution.
+
+SPDX-License-Identifier: Apache-2.0 OR MIT
+"""
+import pytest
+from ly_test_tools.o3de.editor_test import EditorTestSuite, EditorSingleTest
+
+
[email protected]("launcher_platform", ['windows_editor']) # This test works on Windows version of O3DE Editor
[email protected]("project", ["AutomatedTesting"]) # Use AutomatedTesting project
+class TestAutomation(EditorTestSuite):
+    class ShaderAssetBuilder_RGAData(EditorSingleTest):
+        from Atom.tests import RGAtest as test_module

AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_tools_utils.py

@@ -5,15 +5,21 @@ For complete copyright and license terms please see the LICENSE at the root of t
 SPDX-License-Identifier: Apache-2.0 OR MIT
 """
 
+import os
 import pathlib
+import shutil
 import sys
 import time
 
+import azlmbr.asset as azasset
 import azlmbr.atom
 import azlmbr.atomtools
 import azlmbr.bus as bus
+import azlmbr.math as azmath
 import azlmbr.paths
 
+import ly_test_tools.environment.file_system as fs
+
 from Atom.atom_utils.atom_constants import (
     AtomToolsDocumentRequestBusEvents, AtomToolsDocumentSystemRequestBusEvents, AtomToolsMainWindowRequestBusEvents,
     EntityPreviewViewportSettingsRequestBusEvents)
@@ -309,6 +315,62 @@ def wait_for_condition(function: (), timeout_in_seconds: float = 1.0) -> bool or
             if ret:
                 return True
 
+class ShaderAssetTestHelper:
+
+    def copy_file(src_file, src_path, target_file, target_path):
+            # type: (str, str, str, str) -> None
+            """
+            Copies the [src_file] located in [src_path] to the [target_file] located at [target_path].
+            Leaves the [target_file] unlocked for reading and writing privileges
+            :param src_file: The source file to copy (file name)
+            :param src_path: The source file's path
+            :param target_file: The target file to copy into (file name)
+            :param target_path: The target file's path
+            :return: None
+            """
+            target_file_path = os.path.join(target_path, target_file)
+            src_file_path = os.path.join(src_path, src_file)
+            if os.path.exists(target_file_path):
+                fs.unlock_file(target_file_path)
+            shutil.copyfile(src_file_path, target_file_path)
+
+    def copy_tmp_files_in_order(src_directory, file_list, dst_directory, wait_time_in_between=0.0):
+
+        import azlmbr.legacy.general as general
+
+        # type: (str, list, str, float) -> None
+        """
+        This function assumes that for each file name listed in @file_list
+        there's file named "@filename.txt" which the original source file
+        but they will be copied with just the @filename (.txt removed).
+        """
+        for filename in file_list:
+            src_name = f"{filename}.txt"
+            ShaderAssetTestHelper.copy_file(src_name, src_directory, filename, dst_directory)
+            if wait_time_in_between > 0.0:
+                print(f"Created {filename} in {dst_directory}")
+                general.idle_wait(wait_time_in_between)
+
+    def remove_file(src_file, src_path):
+        # type: (str, str) -> None
+        """
+        Removes the [src_file] located in [src_path].
+        :param src_file: The source file to copy (file name)
+        :param src_path: The source file's path
+        :return: None
+        """
+        src_file_path = os.path.join(src_path, src_file)
+        if os.path.exists(src_file_path):
+            fs.unlock_file(src_file_path)
+            os.remove(src_file_path)
+
+    def remove_files(directory, file_list):
+        for filename in file_list:
+            ShaderAssetTestHelper.remove_file(filename, directory)
+
+    def asset_exists(cache_relative_path):
+        asset_id = azasset.AssetCatalogRequestBus(bus.Broadcast, "GetAssetIdByPath", cache_relative_path, azmath.Uuid(), False)
+        return asset_id.is_valid()
 
 class Timeout(object):
     """

AutomatedTesting/Gem/PythonTests/Atom/tests/RGAtest.py

@@ -0,0 +1,92 @@
+"""
+Copyright (c) Contributors to the Open 3D Engine Project.
+For complete copyright and license terms please see the LICENSE at the root of this distribution.
+
+SPDX-License-Identifier: Apache-2.0 OR MIT
+"""
+
+class Tests():
+    rga_file_exist        = ("rga file exist",        "RGA executable doesn't exist! To downlad RGA in O3DE, set O3DE_RADEON_GPU_ANALYZER_ENABLED=true in CmakeCache.txt in build folder, and rerun cmake.")
+    azshader_was_removed  = ("azshader was removed",  "Failed to remove azshader")
+    azshader_was_compiled = ("azshader was compiled", "Failed to compile azshader")
+    livereg_was_created   = ("livereg was created",   "Failed to created livereg")
+    disassem_was_created  = ("disassem was created",  "Failed to created disassem")
+    no_error_occurred     = ("No errors detected",    "Errors were detected")
+
+def RGAtest():
+
+    import sys
+    import os
+
+    from Atom.atom_utils.atom_tools_utils import ShaderAssetTestHelper
+    import azlmbr.legacy.general as general
+    from editor_python_test_tools.utils import TestHelper as helper
+    from editor_python_test_tools.utils import Tracer
+
+    def _file_exists(file_path):
+        return os.path.exists(file_path)
+    
+    # Required for automated tests
+    helper.init_idle()
+
+    game_root_path = os.path.normpath(general.get_game_folder())
+    game_build_path = os.path.normpath(general.get_build_folder())
+    game_asset_path = os.path.join(game_root_path, "Assets")
+    game_cache_path = os.path.join(game_root_path, "Cache")
+    base_dir = os.path.dirname(__file__)
+    src_assets_subdir = os.path.join(base_dir, "TestAssets", "ShaderAssetBuilder")
+
+    with Tracer() as error_tracer:
+        # The script drives the execution of the test, to return the flow back to the editor,
+        # we will tick it one time
+        general.idle_wait_frames(1)
+ 
+        # This is the order in which the source assets should be deployed
+        # to avoid source dependency issues with the old MCPP-based CreateJobs. 
+        file_list = [
+            "RgaShader.shadervariantlist",
+            "RgaShader.shader",
+            "RgaShader.azsl"
+        ]
+        
+        # Check if rga executable exist
+        if sys.platform == 'win32':
+            rga_path = os.path.join(game_build_path, "_deps", "rga-src", "rga.exe")
+        else:
+            rga_path = os.path.join(game_build_path, "_deps", "rga-src", "rga")
+        Report.critical_result(Tests.rga_file_exist, _file_exists(rga_path))
+
+        # Remove files 
+        ShaderAssetTestHelper.remove_files(game_asset_path, file_list)
+
+        # Wait here until the azshader doesn't exist anymore.
+        azshader_name = "assets/rgashader.azshader"
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
+        Report.critical_result(Tests.azshader_was_removed, not ShaderAssetTestHelper.asset_exists(azshader_name))
+
+        ShaderAssetTestHelper.copy_tmp_files_in_order(src_assets_subdir, file_list, game_asset_path)
+
+        # Give enough time to AP to compile the shader
+        helper.wait_for_condition(lambda: ShaderAssetTestHelper.asset_exists(azshader_name), 60.0)
+        Report.critical_result(Tests.azshader_was_compiled, ShaderAssetTestHelper.asset_exists(azshader_name))
+
+        # Check analysis data
+        livereg_path = os.path.normpath(os.path.join(game_cache_path, "pc/assets/gfx1035_livereg_1_frag.txt"))
+        disassem_path = os.path.normpath(os.path.join(game_cache_path, "pc/assets/gfx1035_disassem_1_frag.txt"))
+        helper.wait_for_condition(lambda: _file_exists(livereg_path), 5.0)
+        helper.wait_for_condition(lambda: _file_exists(disassem_path), 5.0)
+        Report.critical_result(Tests.livereg_was_created, _file_exists(livereg_path))
+        Report.critical_result(Tests.disassem_was_created, _file_exists(disassem_path))
+
+        # All good, let's cleanup leftover files before closing the test.
+        ShaderAssetTestHelper.remove_files(game_asset_path, file_list)
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
+
+        # Look for errors to raise.
+        helper.wait_for_condition(lambda: error_tracer.has_errors, 1.0)
+        Report.result(Tests.no_error_occurred, not error_tracer.has_errors)
+
+
+if __name__ == "__main__":
+    from editor_python_test_tools.utils import Report
+    Report.start_test(RGAtest)

AutomatedTesting/Gem/PythonTests/Atom/tests/TestAssets/ShaderAssetBuilder/RgaShader.azsl.txt

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+ /*
+    This is a dummy shader used to validate detection of "#included files"
+ */
+
+#include <Atom/Features/SrgSemantics.azsli>
+
+option bool o_test_option = false;
+
+ShaderResourceGroup DummySrg : SRG_PerDraw
+{
+    float4 m_color;
+}
+
+struct VSInput
+{
+    float3 m_position : POSITION;
+    float4 m_color : COLOR0;
+};
+
+struct VSOutput
+{
+    float4 m_position : SV_Position;
+    float4 m_color : COLOR0;
+};
+
+VSOutput MainVS(VSInput vsInput)
+{
+    VSOutput OUT;
+    OUT.m_position = float4(vsInput.m_position, 1.0);
+    OUT.m_color = vsInput.m_color;
+    return OUT;
+}
+
+struct PSOutput
+{
+    float4 m_color : SV_Target0;
+};
+
+PSOutput MainPS(VSOutput vsOutput)
+{
+    PSOutput OUT;
+
+    OUT.m_color = DummySrg::m_color;
+
+    return OUT;
+}

AutomatedTesting/Gem/PythonTests/Atom/tests/TestAssets/ShaderAssetBuilder/RgaShader.shader.txt

@@ -0,0 +1,26 @@
+// This is a dummy shader used to validate detection of "#included files"
+{
+    "Source" : "RgaShader.azsl",
+
+    "DepthStencilState" : {
+        "Depth" : { "Enable" : false, "CompareFunc" : "GreaterEqual" }
+    },
+
+    "DrawList" : "forward",
+
+    "ProgramSettings":
+    {
+      "EntryPoints":
+      [
+        {
+          "name": "MainVS",
+          "type": "Vertex"
+        },
+        {
+          "name": "MainPS",
+          "type": "Fragment"
+        }
+      ]
+    }
+
+}

AutomatedTesting/Gem/PythonTests/Atom/tests/TestAssets/ShaderAssetBuilder/RgaShader.shadervariantlist.txt

@@ -0,0 +1,12 @@
+{
+    "Shader": "RgaShader.shader",
+    "Variants": [
+        {
+            "StableId": 1,
+            "EnableAnalysis": "true",
+            "Options": {
+                "o_test_option": "true"
+            }
+        }
+    ]
+}

AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges.py

@@ -24,69 +24,11 @@ def ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges():
        at once and We also expect that in the end the shader is built successfully.
     """
     import os
-    import shutil
-
-    import azlmbr.asset as azasset
-    import azlmbr.bus as azbus
     import azlmbr.legacy.general as general
-    import azlmbr.math as azmath
 
+    from Atom.atom_utils.atom_tools_utils import ShaderAssetTestHelper
     from editor_python_test_tools.utils import TestHelper as helper
     from editor_python_test_tools.utils import Tracer
-    import ly_test_tools.environment.file_system as fs
-
-    def _copy_file(src_file, src_path, target_file, target_path):
-        # type: (str, str, str, str) -> None
-        """
-        Copies the [src_file] located in [src_path] to the [target_file] located at [target_path].
-        Leaves the [target_file] unlocked for reading and writing privileges
-        :param src_file: The source file to copy (file name)
-        :param src_path: The source file's path
-        :param target_file: The target file to copy into (file name)
-        :param target_path: The target file's path
-        :return: None
-        """
-        target_file_path = os.path.join(target_path, target_file)
-        src_file_path = os.path.join(src_path, src_file)
-        if os.path.exists(target_file_path):
-            fs.unlock_file(target_file_path)
-        shutil.copyfile(src_file_path, target_file_path)
-
-    def _copy_tmp_files_in_order(src_directory, file_list, dst_directory, wait_time_in_between=0.0):
-        # type: (str, list, str, float) -> None
-        """
-        This function assumes that for each file name listed in @file_list
-        there's file named "@filename.txt" which the original source file
-        but they will be copied with just the @filename (.txt removed).
-        """
-        for filename in file_list:
-            src_name = f"{filename}.txt"
-            _copy_file(src_name, src_directory, filename, dst_directory)
-            if wait_time_in_between > 0.0:
-                print(f"Created {filename} in {dst_directory}")
-                general.idle_wait(wait_time_in_between)
-
-    def _remove_file(src_file, src_path):
-        # type: (str, str) -> None
-        """
-        Removes the [src_file] located in [src_path].
-        :param src_file: The source file to copy (file name)
-        :param src_path: The source file's path
-        :return: None
-        """
-        src_file_path = os.path.join(src_path, src_file)
-        if os.path.exists(src_file_path):
-            fs.unlock_file(src_file_path)
-            os.remove(src_file_path)
-
-    def _remove_files(directory, file_list):
-        for filename in file_list:
-            _remove_file(filename, directory)
-
-    def _asset_exists(cache_relative_path):
-        asset_id = azasset.AssetCatalogRequestBus(azbus.Broadcast, "GetAssetIdByPath", cache_relative_path,
-                                                  azmath.Uuid(), False)
-        return asset_id.is_valid()
 
     # Required for automated tests
     helper.init_idle()
@@ -115,28 +57,28 @@ def ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges():
         reverse_file_list = file_list[::-1]
 
         # Remove files in reverse order
-        _remove_files(game_asset_path, reverse_file_list)
+        ShaderAssetTestHelper.remove_files(game_asset_path, reverse_file_list)
 
         # Wait here until the azshader doesn't exist anymore.
         azshader_name = "assets/dependencyvalidation.azshader"
-        helper.wait_for_condition(lambda: not _asset_exists(azshader_name), 5.0)
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
 
-        Report.critical_result(Tests.azshader_was_removed, not _asset_exists(azshader_name))
+        Report.critical_result(Tests.azshader_was_removed, not ShaderAssetTestHelper.asset_exists(azshader_name))
 
-        _copy_tmp_files_in_order(src_assets_subdir, file_list, game_asset_path, 1.0)
+        ShaderAssetTestHelper.copy_tmp_files_in_order(src_assets_subdir, file_list, game_asset_path, 1.0)
 
         # Give enough time to AP to compile the shader
-        helper.wait_for_condition(lambda: _asset_exists(azshader_name), 60.0)
+        helper.wait_for_condition(lambda: ShaderAssetTestHelper.asset_exists(azshader_name), 60.0)
 
-        Report.critical_result(Tests.azshader_was_compiled, _asset_exists(azshader_name))
+        Report.critical_result(Tests.azshader_was_compiled, ShaderAssetTestHelper.asset_exists(azshader_name))
 
         # The first part was about compiling the shader under normal conditions.
         # Let's remove the files from the previous phase and will proceed
         # to make the source files visible to the AP in reverse order. The
         # ShaderAssetBuilder will only succeed when the last file becomes visible.
-        _remove_files(game_asset_path, reverse_file_list)
-        helper.wait_for_condition(lambda: not _asset_exists(azshader_name), 5.0)
-        Report.critical_result(Tests.azshader_was_removed, not _asset_exists(azshader_name))
+        ShaderAssetTestHelper.remove_files(game_asset_path, reverse_file_list)
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
+        Report.critical_result(Tests.azshader_was_removed, not ShaderAssetTestHelper.asset_exists(azshader_name))
 
         # Remark, if you are running this test manually from the Editor with "pyRunFile",
         # You'll notice how the AP issues notifications that it fails to compile the shader
@@ -149,33 +91,33 @@ def ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges():
         # presented in this test, but with the new version of ShaderAssetBuilder::CreateJobs, which
         # doesn't use MCPP for #include files discovery, it should eventually compile the shader
         # once all the source files are in place.
-        _copy_tmp_files_in_order(src_assets_subdir, reverse_file_list, game_asset_path, 3.0)
+        ShaderAssetTestHelper.copy_tmp_files_in_order(src_assets_subdir, reverse_file_list, game_asset_path, 3.0)
 
         # Give enough time to AP to compile the shader
-        helper.wait_for_condition(lambda: _asset_exists(azshader_name), 60.0)
+        helper.wait_for_condition(lambda: ShaderAssetTestHelper.asset_exists(azshader_name), 60.0)
 
-        Report.critical_result(Tests.azshader_was_compiled, _asset_exists(azshader_name))
+        Report.critical_result(Tests.azshader_was_compiled, ShaderAssetTestHelper.asset_exists(azshader_name))
 
         # The last phase of the test puts stress on potential race conditions
         # when all required files appear as soon as possible.
 
         # First Clean up.
         # Remove left over files.
-        _remove_files(game_asset_path, reverse_file_list)
-        helper.wait_for_condition(lambda: not _asset_exists(azshader_name), 5.0)
-        Report.critical_result(Tests.azshader_was_removed, not _asset_exists(azshader_name))
+        ShaderAssetTestHelper.remove_files(game_asset_path, reverse_file_list)
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
+        Report.critical_result(Tests.azshader_was_removed, not ShaderAssetTestHelper.asset_exists(azshader_name))
 
         # Now let's copy all the source files to the "Assets" folder as fast as possible.
-        _copy_tmp_files_in_order(src_assets_subdir, reverse_file_list, game_asset_path)
+        ShaderAssetTestHelper.copy_tmp_files_in_order(src_assets_subdir, reverse_file_list, game_asset_path)
 
         # Give enough time to AP to compile the shader
-        helper.wait_for_condition(lambda: _asset_exists(azshader_name), 60.0)
+        helper.wait_for_condition(lambda: ShaderAssetTestHelper.asset_exists(azshader_name), 60.0)
 
-        Report.critical_result(Tests.azshader_was_compiled, _asset_exists(azshader_name))
+        Report.critical_result(Tests.azshader_was_compiled, ShaderAssetTestHelper.asset_exists(azshader_name))
 
         # All good, let's cleanup leftover files before closing the test.
-        _remove_files(game_asset_path, reverse_file_list)
-        helper.wait_for_condition(lambda: not _asset_exists(azshader_name), 5.0)
+        ShaderAssetTestHelper.remove_files(game_asset_path, reverse_file_list)
+        helper.wait_for_condition(lambda: not ShaderAssetTestHelper.asset_exists(azshader_name), 5.0)
 
         # Look for errors to raise.
         helper.wait_for_condition(lambda: error_tracer.has_errors, 1.0)

Code/Editor/CryEditPy.cpp

@@ -17,6 +17,7 @@
 // AzCore
 #include <AzCore/Component/TickBus.h>
 #include <AzCore/Console/IConsole.h>
+#include <AzCore/Utils/Utils.h>
 
 // AzToolsFramework
 #include <AzToolsFramework/API/EditorPythonConsoleBus.h>
@@ -97,6 +98,14 @@ namespace
         return Path::GetEditingGameDataFolder();
     }
 
+    AZStd::string PyGetBuildFolderAsString()
+    {
+        AZ::IO::FixedMaxPath projectBuildPath = AZ::Utils::GetExecutableDirectory();
+        projectBuildPath = projectBuildPath.RemoveFilename(); // profile
+        projectBuildPath = projectBuildPath.RemoveFilename(); // bin
+        return AZStd::string(projectBuildPath.c_str());
+    }
+
     bool PyOpenLevel(const char* pLevelName)
     {
         const char* oldExtension = EditorUtils::LevelFile::GetOldCryFileExtension();
@@ -401,6 +410,7 @@ namespace AzToolsFramework
             addLegacyGeneral(behaviorContext->Method("create_level", ::PyCreateLevel, nullptr, "Creates a level with the parameters of 'levelName', 'resolution', 'unitSize' and 'bUseTerrain'."));
             addLegacyGeneral(behaviorContext->Method("create_level_no_prompt", ::PyCreateLevelNoPrompt, nullptr, "Creates a level with the parameters of 'levelName', 'resolution', 'unitSize' and 'bUseTerrain'."));
             addLegacyGeneral(behaviorContext->Method("get_game_folder", PyGetGameFolderAsString, nullptr, "Gets the path to the Game folder of current project."));
+            addLegacyGeneral(behaviorContext->Method("get_build_folder", PyGetBuildFolderAsString, nullptr, "Gets the build folder path of current project."));
             addLegacyGeneral(behaviorContext->Method("get_current_level_name", PyGetCurrentLevelName, nullptr, "Gets the name of the current level."));
             addLegacyGeneral(behaviorContext->Method("get_current_level_path", PyGetCurrentLevelPath, nullptr, "Gets the fully specified path of the current level."));