Add test target console output routing.

Signed-off-by: John <[email protected]>

Code/Tools/TestImpactFramework/Frontend/Console/Common/Code/Source/TestImpactCommandLineOptions.cpp

@@ -477,7 +477,7 @@ namespace TestImpact
             "                                                                them to be drafted into future test runs and 'keep' will keep any existing \n"
             "                                                                coverage data and update the coverage data for failed tests that produce \n"
             "                                                                coverage.\n"
-            "    -targetout=<sdtout, file>                                   Capture of individual test run stdout, where 'stdout' will capture \n"
+            "    -targetout=<stdout, file>                                   Capture of individual test run stdout, where 'stdout' will capture \n"
             "                                                                each individual test target's stdout and output each one to stdout \n"
             "                                                                and 'file' will capture each individual test target's stdout and output \n"
             "                                                                each one individually to a file (multiple values are accepted).\n"

Code/Tools/TestImpactFramework/Frontend/Console/Common/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp

@@ -17,6 +17,11 @@ namespace TestImpact
 {
     namespace Console
     {
+        static void PrintDivider()
+        {
+            std::cout << "-----------------------------------------------------------------------------\n";
+        }
+
         namespace Output
         {
             void TestSuiteFilter(SuiteType filter)
@@ -114,6 +119,8 @@ namespace TestImpact
             Output::TestSuiteFilter(suiteType);
             std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns()
                       << " excluded.\n";
+
+            PrintDivider();
         }
 
         void TestSequenceCompleteCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests)
@@ -131,6 +138,8 @@ namespace TestImpact
             Output::TestSuiteFilter(suiteType);
             Output::ImpactAnalysisTestSelection(
                 selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size());
+
+            PrintDivider();
         }
 
         void SafeImpactAnalysisTestSequenceStartCallback(
@@ -145,6 +154,8 @@ namespace TestImpact
                 discardedTests.GetTotalNumTests(),
                 selectedTests.GetNumExcludedTestRuns() + discardedTests.GetNumExcludedTestRuns(),
                 draftedTests.size());
+
+            PrintDivider();
         }
 
         void RegularTestSequenceCompleteCallback(const Client::RegularSequenceReport& sequenceReport)
@@ -188,6 +199,16 @@ namespace TestImpact
             const auto progress =
                 AZStd::string::format("(%03zu/%03zu)", numTestRunsCompleted, totalNumTestRuns);
 
+            if (not testRun.GetStdOutput().empty())
+            {
+                std::cout << testRun.GetStdOutput().c_str();
+            }
+
+            if (not testRun.GetStdError().empty())
+            {
+                std::cout << testRun.GetStdError().c_str();
+            }
+
             AZStd::string result;
             switch (testRun.GetResult())
             {
@@ -224,6 +245,8 @@ namespace TestImpact
 
             std::cout << progress.c_str() << " " << result.c_str() << " " << testRun.GetTargetName().c_str() << " ("
                       << (testRun.GetDuration().count() / 1000.f) << "s)\n";
+
+            PrintDivider();
         }
     } // namespace Console
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Common/Code/Include/Headers/TestImpactFramework/TestImpactClientTestRun.h

@@ -40,6 +40,8 @@ namespace TestImpact
                 const AZStd::string& testNamespace,
                 const AZStd::string& name,
                 const AZStd::string& commandString,
+                const AZStd::string& stdOutput,
+                const AZStd::string& stdError,
                 AZStd::chrono::steady_clock::time_point startTime,
                 AZStd::chrono::milliseconds duration,
                 TestRunResult result);
@@ -55,6 +57,12 @@ namespace TestImpact
             //! Returns the test run result.
             TestRunResult GetResult() const;
 
+            //! Returns the standard output produced by this test run.
+            const AZStd::string& GetStdOutput() const;
+
+            //! Returns the standard error produced by this test run.
+            const AZStd::string& GetStdError() const;
+
             //! Returns the test run start time.
             AZStd::chrono::steady_clock::time_point GetStartTime() const;
 
@@ -72,6 +80,8 @@ namespace TestImpact
             AZStd::string m_commandString;
             AZStd::string m_testNamespace;
             TestRunResult m_result;
+            AZStd::string m_stdOutput;
+            AZStd::string m_stdError;
             AZStd::chrono::steady_clock::time_point m_startTime;
             AZStd::chrono::milliseconds m_duration;
         };
@@ -144,6 +154,8 @@ namespace TestImpact
             CompletedTestRun(
                 const AZStd::string& name,
                 const AZStd::string& commandString,
+                const AZStd::string& stdOutput,
+                const AZStd::string& stdError,
                 AZStd::chrono::steady_clock::time_point startTime,
                 AZStd::chrono::milliseconds duration,
                 TestRunResult result,

Code/Tools/TestImpactFramework/Runtime/Common/Code/Include/Static/TestImpactRuntimeUtils.h

@@ -72,6 +72,8 @@ namespace TestImpact
                     testJob.GetTestTarget()->GetNamespace(),
                     testJob.GetTestTarget()->GetName(),
                     testJob.GetCommandString(),
+                    testJob.GetStdOutput(),
+                    testJob.GetStdError(),
                     testJob.GetStartTime(),
                     testJob.GetDuration(),
                     testJob.GetTestResult());
@@ -337,6 +339,8 @@ namespace TestImpact
                 testJob.GetTestTarget()->GetNamespace(),
                 testJob.GetTestTarget()->GetName(),
                 testJob.GetCommandString(),
+                testJob.GetStdOutput(),
+                testJob.GetStdError(),
                 relativeStartTime,
                 testJob.GetDuration(),
                 testJob.GetTestResult());

Code/Tools/TestImpactFramework/Runtime/Common/Code/Source/TestImpactClientSequenceReportSerializer.cpp

@@ -638,9 +638,11 @@ namespace TestImpact
     Client::TestRunBase DeserializeTestRunBase(const rapidjson::Value& serialTestRun)
     {
         return Client::TestRunBase(
-            "",
+            "", // Namespace
             serialTestRun[SequenceReportFields::Keys[SequenceReportFields::Name]].GetString(),
             serialTestRun[SequenceReportFields::Keys[SequenceReportFields::CommandArgs]].GetString(),
+            "", // StdOut
+            "", // StdError
             TimePointFromMsInt64(serialTestRun[SequenceReportFields::Keys[SequenceReportFields::StartTime]].GetInt64()),
             AZStd::chrono::milliseconds(serialTestRun[SequenceReportFields::Keys[SequenceReportFields::Duration]].GetInt64()),
             TestRunResultFromString(serialTestRun[SequenceReportFields::Keys[SequenceReportFields::Result]].GetString()));

Code/Tools/TestImpactFramework/Runtime/Common/Code/Source/TestImpactClientTestRun.cpp

@@ -18,11 +18,15 @@ namespace TestImpact
             const AZStd::string& testNamespace,
             const AZStd::string& name,
             const AZStd::string& commandString,
+            const AZStd::string& stdOutput,
+            const AZStd::string& stdError,
             AZStd::chrono::steady_clock::time_point startTime,
             AZStd::chrono::milliseconds duration,
             TestRunResult result)
             : m_targetName(name)
             , m_commandString(commandString)
+            , m_stdOutput(stdOutput)
+            , m_stdError(stdError)
             , m_startTime(startTime)
             , m_duration(duration)
             , m_result(result)
@@ -65,6 +69,16 @@ namespace TestImpact
             return m_result;
         }
 
+        const AZStd::string& TestRunBase::GetStdOutput() const
+        {
+            return m_stdOutput;
+        }
+
+        const AZStd::string& TestRunBase::GetStdError() const
+        {
+            return m_stdError;
+        }
+
         TestRunWithExecutionFailure::TestRunWithExecutionFailure(TestRunBase&& testRun)
             : TestRunBase(AZStd::move(testRun))
         {
@@ -124,12 +138,14 @@ namespace TestImpact
         CompletedTestRun::CompletedTestRun(
             const AZStd::string& name,
             const AZStd::string& commandString,
+            const AZStd::string& stdOutput,
+            const AZStd::string& stdError,
             AZStd::chrono::steady_clock::time_point startTime,
             AZStd::chrono::milliseconds duration,
             TestRunResult result,
             AZStd::vector<Test>&& tests,
             const AZStd::string& testNamespace)
-            : TestRunBase(testNamespace, name, commandString, startTime, duration, result)
+            : TestRunBase(testNamespace, name, commandString, stdOutput, stdError, startTime, duration, result)
             , m_tests(AZStd::move(tests))
         {
             AZStd::tie(m_totalNumPassingTests, m_totalNumFailingTests, m_totalNumDisabledTests) = CalculateTestCaseMetrics(m_tests);

Code/Tools/TestImpactFramework/Runtime/Python/Code/Source/TestEngine/Python/TestImpactPythonTestEngine.cpp

@@ -40,13 +40,13 @@ namespace TestImpact
     template<>
     struct TestJobRunnerTrait<PythonTestRunner>
     {
-        using TestEngineJobType = TestEngineInstrumentedRun<typename PythonTestEngine::TestTargetType, typename PythonTestEngine::TestCaseCoverageType>;
+        using TestEngineJobType = TestEngineInstrumentedRun<PythonTestTarget, TestCoverage>;
     };
 
     template<>
     struct TestJobRunnerTrait<PythonNullTestRunner>
     {
-        using TestEngineJobType = TestEngineInstrumentedRun<typename PythonTestEngine::TestTargetType, typename PythonTestEngine::TestCaseCoverageType>;
+        using TestEngineJobType = TestEngineInstrumentedRun<PythonTestTarget, TestCoverage>;
     };
 
     PythonTestEngine::PythonTestEngine(
@@ -71,39 +71,18 @@ namespace TestImpact
         DeleteFiles(m_artifactDir.m_coverageArtifactDirectory, "*.pycoverage");
     }
 
-    TestEngineInstrumentedRunResult<typename PythonTestEngine::TestTargetType, typename PythonTestEngine::TestCaseCoverageType>
+    TestEngineInstrumentedRunResult<PythonTestTarget, TestCoverage>
         PythonTestEngine::
         InstrumentedRun(
-        [[maybe_unused]] const AZStd::vector<const PythonTestTarget*>& testTargets,
-        [[maybe_unused]] Policy::ExecutionFailure executionFailurePolicy,
-        [[maybe_unused]] Policy::IntegrityFailure integrityFailurePolicy,
-        [[maybe_unused]] Policy::TestFailure testFailurePolicy,
-        [[maybe_unused]] Policy::TargetOutputCapture targetOutputCapture,
-        [[maybe_unused]] AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
-        [[maybe_unused]] AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-        [[maybe_unused]] AZStd::optional<TestEngineJobCompleteCallback<PythonTestTarget>> callback) const
+        const AZStd::vector<const PythonTestTarget*>& testTargets,
+        Policy::ExecutionFailure executionFailurePolicy,
+        Policy::IntegrityFailure integrityFailurePolicy,
+        Policy::TestFailure testFailurePolicy,
+        Policy::TargetOutputCapture targetOutputCapture,
+        AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
+        AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
+        AZStd::optional<TestEngineJobCompleteCallback<PythonTestTarget>> callback) const
     {
-        // We currently don't have a std out/error callback for the test engine users so output the Python
-        // error and output here for the time being
-        const auto stdPrint = []([[maybe_unused]] const typename PythonNullTestRunner::JobInfo& jobInfo,
-                                 [[maybe_unused]] const AZStd::string& stdOutput,
-                                 [[maybe_unused]] const AZStd::string& stdError,
-                                 AZStd::string&& stdOutDelta,
-                                 [[maybe_unused]] AZStd::string&& stdErrDelta)
-        {
-            if (!stdOutDelta.empty())
-            {
-                std::cout << stdOutDelta.c_str() << "\n";
-            }
-
-            if (!stdErrDelta.empty())
-            {
-                std::cout << stdErrDelta.c_str() << "\n";
-            }
-
-            return TestImpact::ProcessCallbackResult::Continue;
-        };
-
         if (m_testRunnerPolicy == Policy::TestRunner::UseNullTestRunner)
         {
             // We don't delete the artifacts as they have been left by another test runner (e.g. ctest)
@@ -119,7 +98,7 @@ namespace TestImpact
                 testTargetTimeout,
                 globalTimeout,
                 callback,
-                stdPrint),
+                std::nullopt), // For real-time stdout/err output of test targets
             integrityFailurePolicy);
         }
         else
@@ -137,7 +116,7 @@ namespace TestImpact
                     testTargetTimeout,
                     globalTimeout,
                     callback,
-                    stdPrint),
+                    std::nullopt), // For real-time stdout/err output of test targets
                 integrityFailurePolicy);
         }
     }

Code/Tools/TestImpactFramework/Runtime/Python/Code/Source/TestEngine/Python/TestImpactPythonTestEngine.h

@@ -32,8 +32,7 @@ namespace TestImpact
     class PythonTestEngine
     {
     public:
-        using TestTargetType = PythonTestTarget;
-        using TestCaseCoverageType = TestCoverage;
+        using TestTarget = PythonTestTarget;
 
         //! Configures the test engine with the necessary path information for launching test targets and managing the artifacts they
         //! produce.
@@ -62,7 +61,7 @@ namespace TestImpact
         //! empty).
         //! @param callback The client callback function to handle completed test target runs.
         //! @ returns The sequence result and the test run results and test coverages for the test targets that were run.
-        [[nodiscard]] TestEngineInstrumentedRunResult<TestTargetType, TestCaseCoverageType>
+        [[nodiscard]] TestEngineInstrumentedRunResult<TestTarget, TestCoverage>
         InstrumentedRun(
             const AZStd::vector<const PythonTestTarget*>& testTargets,
             Policy::ExecutionFailure executionFailurePolicy,

scripts/build/Platform/Windows/build_config.json

@@ -77,7 +77,7 @@
       "CONFIGURATION": "profile",
       "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py",
       "SCRIPT_PARAMETERS": 
-      "--config=\"%OUTPUT_DIRECTORY%/bin/TestImpactFramework/profile/Persistent/tiaf.json\" --src-branch=%BRANCH_NAME% --dst-branch=%CHANGE_TARGET% --commit=%CHANGE_ID% --s3-bucket=%TEST_IMPACT_S3_BUCKET% --mars-index-prefix=o3de-tiaf --s3-top-level-dir=%REPOSITORY_NAME% --build-number=%BUILD_NUMBER% --suite=main --test-failure-policy=continue --runtime-type=python --testrunner=live"
+      "--config=\"%OUTPUT_DIRECTORY%/bin/TestImpactFramework/profile/Persistent/tiaf.json\" --src-branch=%BRANCH_NAME% --dst-branch=%CHANGE_TARGET% --commit=%CHANGE_ID% --s3-bucket=%TEST_IMPACT_S3_BUCKET% --mars-index-prefix=o3de-tiaf --s3-top-level-dir=%REPOSITORY_NAME% --build-number=%BUILD_NUMBER% --suite=main --test-failure-policy=continue --runtime-type=python --testrunner=live --target-output=stdout"
     }
   },
   "debug": {

scripts/build/TestImpactAnalysis/test_impact/runtime_test_impact_args.py

@@ -34,6 +34,8 @@ class RuntimeArgs(Enum):
                "Integration failure policy is set to: ")
     COMMON_CHANGELIST = ("change_list", "--changelist=", "Change list is set to: ")
     COMMON_REPORT = ("report", "--report=", "Sequencer report file is set to: ")
+    COMMON_TARGET_OUTPUT = ("target_output", "--targetout=",
+                      "Test target output capture is set to: ")
 
     # Native arguments
     NATIVE_SAFEMODE = ("safe_mode", "--safemode=", "Safe mode set to: ")

scripts/build/TestImpactAnalysis/tiaf_driver.py

@@ -137,6 +137,7 @@ def parse_args():
         required=False
     )
 
+    # Target exclusion JSON file
     parser.add_argument(
         '--exclude-file',
         type=valid_file_path,
@@ -144,6 +145,7 @@ def parse_args():
         required=False
     )
 
+    # Runtime type (native or python)
     parser.add_argument(
         "--runtime-type",
         choices=SUPPORTED_RUNTIMES.keys(),
@@ -151,16 +153,26 @@ def parse_args():
         required=True
     )
 
+    # Runtime sequence override
     parser.add_argument(
         "--sequence-override",
         help="Override sequence type",
         required=False
     )
 
+    # Python test runner policy
     parser.add_argument(
         "--testrunner-policy",
         choices=["live","null"],
-        help="Test runner policy for TIAF.",
+        help="Test runner policy for TIAF",
+        required=False
+    )
+
+    # Test target output routing
+    parser.add_argument(
+        "--target-output",
+        choices=["stdout"],
+        help="Test target std/error output routing",
         required=False
     )