Added random seed for the test execution in full test suite

Gem/Code/Source/Automation/ScriptManager.cpp

@@ -448,11 +448,12 @@ namespace AtomSampleViewer
         m_showScriptRunnerDialog = true;
     }
 
-    void ScriptManager::RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd)
+    void ScriptManager::RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd, int randomSeed)
     {
         m_testSuiteRunConfig.m_automatedRunEnabled = true;
         m_testSuiteRunConfig.m_testSuitePath = suiteFilePath;
         m_testSuiteRunConfig.m_closeOnTestScriptFinish = exitOnTestEnd;
+        m_testSuiteRunConfig.m_randomSeed = randomSeed;
     }
 
     void ScriptManager::AbortScripts(const AZStd::string& reason)
@@ -508,6 +509,8 @@ namespace AtomSampleViewer
             ImGui::Text("Settings");
             ImGui::Indent();
 
+            ImGui::InputInt("Random Seed for Test Order Execution", &m_testSuiteRunConfig.m_randomSeed);
+
             m_imageComparisonOptions.DrawImGuiSettings();
             if (ImGui::Button("Reset"))
             {
@@ -733,6 +736,7 @@ namespace AtomSampleViewer
         behaviorContext->Method("NormalizePath", &Script_NormalizePath);
         behaviorContext->Method("DegToRad", &Script_DegToRad);
         behaviorContext->Method("GetRenderApiName", &Script_GetRenderApiName);
+        behaviorContext->Method("GetRandomTestSeed", &Script_GetRandomTestSeed);
 
         // Samples...
         behaviorContext->Method("OpenSample", &Script_OpenSample);
@@ -1375,6 +1379,11 @@ namespace AtomSampleViewer
         
     }
 
+    int ScriptManager::Script_GetRandomTestSeed()
+    {
+        return s_instance->m_testSuiteRunConfig.m_randomSeed;
+    }
+
     void ScriptManager::CheckArcBallControllerHandler()
     {
         if (0 == AZ::Debug::ArcBallControllerRequestBus::GetNumOfEventHandlers(s_instance->m_cameraEntity->GetId()))

Gem/Code/Source/Automation/ScriptManager.h

@@ -67,7 +67,7 @@ namespace AtomSampleViewer
 
         void OpenScriptRunnerDialog();
 
-        void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd);
+        void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd, int randomSeed);
         
     private:
 
@@ -99,6 +99,7 @@ namespace AtomSampleViewer
         static AZStd::string Script_NormalizePath(const AZStd::string& path);
         static float Script_DegToRad(float degrees);
         static AZStd::string Script_GetRenderApiName();
+        static int Script_GetRandomTestSeed();
 
         // Samples...
         static void Script_OpenSample(const AZStd::string& sampleName);
@@ -223,6 +224,7 @@ namespace AtomSampleViewer
             bool m_isStarted = false;
             bool m_closeOnTestScriptFinish = false;
             AZStd::string m_testSuitePath;
+            int m_randomSeed;
         };
 
         TestSuiteExecutionConfig m_testSuiteRunConfig;

Gem/Code/Source/SampleComponentManager.cpp

@@ -1118,11 +1118,11 @@ namespace AtomSampleViewer
         return m_isFrameCapturePending;
     }
 
-    void SampleComponentManager::RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd)
+    void SampleComponentManager::RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd, int randomSeed)
     {
         if (m_scriptManager)
         {
-            m_scriptManager->RunMainTestSuite(suiteFilePath, exitOnTestEnd);
+            m_scriptManager->RunMainTestSuite(suiteFilePath, exitOnTestEnd, randomSeed);
         }
     }
 

Gem/Code/Source/SampleComponentManager.h

@@ -147,7 +147,7 @@ namespace AtomSampleViewer
         bool ShowTool(const AZStd::string& toolName, bool enable) override;
         void RequestFrameCapture(const AZStd::string& filePath, bool hideImGui) override;
         bool IsFrameCapturePending() override;
-        void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd) override;
+        void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd, int randomSeed) override;
 
         // FrameCaptureNotificationBus overrides...
         void OnCaptureFinished(AZ::Render::FrameCaptureResult result, const AZStd::string& info) override;

Gem/Code/Source/SampleComponentManagerBus.h

@@ -40,7 +40,8 @@ namespace AtomSampleViewer
         //! Runs the main test suite, in an automated mode and closes AtomSampleViewer when the suite finished running
         //! @param suiteFilePath path to the compiled luac test script
         //! @param exitOnTestEnd if true, exits AtomSampleViewerStandalone when the script finishes, used in jenkins
-        virtual void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd) = 0;
+        //! @param randomSeed the seed for the random generator, frequently used inside lua tests to shuffle the order of the test execution
+        virtual void RunMainTestSuite(const AZStd::string& suiteFilePath, bool exitOnTestEnd, int randomSeed) = 0;
     };
     using SampleComponentManagerRequestBus = AZ::EBus<SampleComponentManagerRequests>;
 

Scripts/_FullTestSuite_.bv.lua

@@ -14,76 +14,102 @@
 
 -- Fast check for a sample which doesn't have a dedicated test script
 function FastCheckSample(sampleName)
-    Print("========= Begin Fast-check " .. sampleName .. " =========")
-    OpenSample(sampleName)
-    IdleSeconds(2) 
-    OpenSample(nil)
-    Print("========= End Fast-check " .. sampleName .. " =========")
+    return function()
+        Print("========= Begin Fast-check " .. sampleName .. " =========")
+        OpenSample(sampleName)
+        IdleSeconds(2) 
+        OpenSample(nil)
+        Print("========= End Fast-check " .. sampleName .. " =========")
+    end
 end
 
-RunScript('scripts/decals.bv.luac')
-RunScript('scripts/dynamicdraw.bv.luac')
-RunScript('scripts/dynamicmaterialtest.bv.luac')
-RunScript('scripts/materialscreenshottests.bv.luac')
-RunScript('scripts/materialhotreloadtest.bv.luac')
-RunScript('scripts/msaa_rpi_test.bv.luac')
-RunScript('scripts/cullingandlod.bv.luac')
-RunScript('scripts/multirenderpipeline.bv.luac')
-RunScript('scripts/lightculling.bv.luac')
-RunScript('scripts/transparenttest.bv.luac')
-RunScript('scripts/streamingimagetest.bv.luac')
-RunScript('scripts/parallaxtest.bv.luac')
-RunScript('scripts/checkerboardtest.bv.luac')
-RunScript('scripts/scenereloadsoaktest.bv.luac')
-RunScript('scripts/diffusegitest.bv.luac')
-RunScript('scripts/arealighttest.bv.luac')
-RunScript('scripts/multiscene.bv.luac')
-RunScript('scripts/shadowtest.bv.luac')
-RunScript('scripts/shadowedbistrotest.bv.luac')
-RunScript('scripts/PassTree.bv.luac')
+-- Test helper functions
+function random_shuffle(list)
+    for i = #list, 2, -1 do
+        local j = math.random(i)
+        list[i], list[j] = list[j], list[i]
+    end
+end
+
+function RunScriptWrapper(name)
+    return function() RunScript(name) end
+end
+
+tests= {
+    RunScriptWrapper('scripts/decals.bv.luac'),
+    RunScriptWrapper('scripts/dynamicdraw.bv.luac'),
+    RunScriptWrapper('scripts/dynamicmaterialtest.bv.luac'),
+    RunScriptWrapper('scripts/materialscreenshottests.bv.luac'),
+    RunScriptWrapper('scripts/materialhotreloadtest.bv.luac'),
+    -- RunScriptWrapper('scripts/msaa_rpi_test.bv.luac'), -- seems broken at the moment?
+    RunScriptWrapper('scripts/cullingandlod.bv.luac'),
+    RunScriptWrapper('scripts/multirenderpipeline.bv.luac'),
+    RunScriptWrapper('scripts/lightculling.bv.luac'),
+    RunScriptWrapper('scripts/transparenttest.bv.luac'),
+    RunScriptWrapper('scripts/streamingimagetest.bv.luac'),
+    RunScriptWrapper('scripts/parallaxtest.bv.luac'),
+    RunScriptWrapper('scripts/checkerboardtest.bv.luac'),
+    RunScriptWrapper('scripts/scenereloadsoaktest.bv.luac'),
+    RunScriptWrapper('scripts/diffusegitest.bv.luac'),
+    RunScriptWrapper('scripts/arealighttest.bv.luac'),
+    RunScriptWrapper('scripts/multiscene.bv.luac'),
+    RunScriptWrapper('scripts/shadowtest.bv.luac'),
+    RunScriptWrapper('scripts/shadowedbistrotest.bv.luac'),
+    RunScriptWrapper('scripts/PassTree.bv.luac'),
+
+    --Fast checking for the samples which don't have a test. Samples should be removed from this list once they have their own tests
 
---Fast checking for the samples which don't have a test. Samples should be removed from this list once they have their own tests
+    FastCheckSample('RHI/AlphaToCoverage'),
+    FastCheckSample('RHI/AsyncCompute'),
+    FastCheckSample('RHI/BindlessPrototype'),
+    FastCheckSample('RHI/Compute'),
+    FastCheckSample('RHI/CopyQueue'),
+    FastCheckSample('RHI/DualSourceBlending'),
+    FastCheckSample('RHI/IndirectRendering'),
+    FastCheckSample('RHI/InputAssembly'),
+    FastCheckSample('RHI/MSAA'),
+    FastCheckSample('RHI/MultipleViews'),
+    FastCheckSample('RHI/MultiRenderTarget'),
+    FastCheckSample('RHI/MultiThread'),
+    FastCheckSample('RHI/Queries'),
+    FastCheckSample('RHI/SphericalHarmonics'),
+    FastCheckSample('RHI/Stencil'),
+    FastCheckSample('RHI/Swapchain'),
+    FastCheckSample('RHI/Texture'),
+    FastCheckSample('RHI/Texture3d'),
+    -- FastCheckSample('RHI/TextureArray'), -- seems broken at the moment?
+    FastCheckSample('RHI/TextureMap'),
+    FastCheckSample('RHI/Triangle'),
+    FastCheckSample('RHI/TrianglesConstantBuffer'),
+
+    FastCheckSample('RPI/AssetLoadTest'),
+    FastCheckSample('RPI/AuxGeom'),
+    FastCheckSample('RPI/BistroBenchmark'),
+    FastCheckSample('RPI/MultiViewSingleSceneAuxGeom'),
+    FastCheckSample('RPI/RootConstants'),
+    FastCheckSample('RPI/Shading'),
+
+    FastCheckSample('Features/Bloom'),
+    FastCheckSample('Features/DepthOfField'),
+    -- FastCheckSample('Features/Exposure'), -- seems broken at the moment?
+    FastCheckSample('Features/SkinnedMesh'),
+    FastCheckSample('Features/SSAO'),
+    FastCheckSample('Features/SSR'),
+    FastCheckSample('Features/Tonemapping'),
+}
 
-FastCheckSample('RHI/AlphaToCoverage')
-FastCheckSample('RHI/AsyncCompute')
-FastCheckSample('RHI/BindlessPrototype')
-FastCheckSample('RHI/Compute')
-FastCheckSample('RHI/CopyQueue')
-FastCheckSample('RHI/DualSourceBlending')
-FastCheckSample('RHI/IndirectRendering')
-FastCheckSample('RHI/InputAssembly')
-FastCheckSample('RHI/MSAA')
-FastCheckSample('RHI/MultipleViews')
-FastCheckSample('RHI/MultiRenderTarget')
-FastCheckSample('RHI/MultiThread')
-FastCheckSample('RHI/Queries')
 if (GetRenderApiName() == "dx12") then 
-    FastCheckSample('RHI/RayTracing')
+    table.insert(tests, FastCheckSample('RHI/RayTracing'))
 end
-FastCheckSample('RHI/SphericalHarmonics')
-FastCheckSample('RHI/Stencil')
+
 if (GetRenderApiName() ~= "dx12") then 
-    FastCheckSample('RHI/Subpass') 
+    table.insert(tests, FastCheckSample('RHI/Subpass'))
 end
-FastCheckSample('RHI/Swapchain')
-FastCheckSample('RHI/Texture')
-FastCheckSample('RHI/Texture3d')
-FastCheckSample('RHI/TextureArray')
-FastCheckSample('RHI/TextureMap')
-FastCheckSample('RHI/Triangle')
-FastCheckSample('RHI/TrianglesConstantBuffer')
 
-FastCheckSample('RPI/AssetLoadTest')
-FastCheckSample('RPI/AuxGeom')
-FastCheckSample('RPI/BistroBenchmark')
-FastCheckSample('RPI/MultiViewSingleSceneAuxGeom')
-FastCheckSample('RPI/RootConstants')
-FastCheckSample('RPI/Shading')
-
-FastCheckSample('Features/Bloom')
-FastCheckSample('Features/DepthOfField')
-FastCheckSample('Features/Exposure')
-FastCheckSample('Features/SkinnedMesh')
-FastCheckSample('Features/SSAO')
-FastCheckSample('Features/SSR')
-FastCheckSample('Features/Tonemapping')
+seed = GetRandomTestSeed()
+Print("========= Using " .. seed .. " as a random seed to sort the tests =========")
+math.randomseed(seed) -- pass a custom seed here
+random_shuffle(tests)
+for k,test in pairs(tests) do
+    test()
+end

Standalone/Platform/Common/AtomSampleViewerApplication.cpp

@@ -27,6 +27,7 @@
 #include <GridMate/Drillers/ReplicaDriller.h>
 
 #include <Atom/RPI.Public/RPISystemInterface.h>
+#include <AzCore/Math/Random.h>
 #include <AzCore/Script/ScriptAsset.h>
 #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzCore/Utils/Utils.h>
@@ -54,7 +55,7 @@ namespace AtomSampleViewer
         AZ::Debug::TraceMessageBus::Handler::BusConnect();
         AtomSampleViewerRequestsBus::Handler::BusConnect();
         SampleComponentManagerNotificationBus::Handler::BusConnect();
-        
+
         AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddBuildSystemTargetSpecialization(
             *AZ::SettingsRegistry::Get(), GetBuildTargetName());
     }
@@ -73,7 +74,7 @@ namespace AtomSampleViewer
         SampleComponentManagerNotificationBus::Handler::BusConnect();
 
         SetupConsoleHandlerRoutine();
-        
+
         AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddBuildSystemTargetSpecialization(
             *AZ::SettingsRegistry::Get(), GetBuildTargetName());
     }
@@ -165,13 +166,21 @@ namespace AtomSampleViewer
 
             constexpr const char* testSuiteSwitch = "runtestsuite";
             constexpr const char* testExitSwitch = "exitontestend";
+            constexpr const char* testRandomSeed = "randomtestseed";
 
             bool exitOnTestEnd = commandLine.HasSwitch(testExitSwitch);
 
             if (commandLine.HasSwitch(testSuiteSwitch))
             {
                 const AZStd::string& testSuitePath = commandLine.GetSwitchValue(testSuiteSwitch, 0);
-                SampleComponentManagerRequestBus::Broadcast(&SampleComponentManagerRequestBus::Events::RunMainTestSuite, testSuitePath, exitOnTestEnd);
+
+                int randomSeed = AZ::SimpleLcgRandom().GetRandom();
+                if (commandLine.HasSwitch(testRandomSeed))
+                {
+                    randomSeed = atoi(commandLine.GetSwitchValue(testRandomSeed, 0).c_str());
+                }
+
+                SampleComponentManagerRequestBus::Broadcast(&SampleComponentManagerRequestBus::Events::RunMainTestSuite, testSuitePath, exitOnTestEnd, randomSeed);
             }
         }
     }