|
|
@@ -110,7 +110,8 @@ namespace TestImpact
|
|
|
const auto sparTiaData = SerializeSourceCoveringTestsList(sparTia);
|
|
|
WriteFileContents<RuntimeException>(sparTiaData, sparTiaFile);
|
|
|
return true;
|
|
|
- } catch (const RuntimeException& e)
|
|
|
+ }
|
|
|
+ catch (const RuntimeException& e)
|
|
|
{
|
|
|
if (integrationFailurePolicy == Policy::IntegrityFailure::Abort)
|
|
|
{
|
|
|
@@ -253,40 +254,6 @@ namespace TestImpact
|
|
|
return AZStd::make_unique<TestTargetExclusionList<TestTarget>>(AZStd::move(testTargetExcludeList));
|
|
|
}
|
|
|
|
|
|
- //! Selects the test targets from the specified list of test targets that are not in the specified test target exclusion list.
|
|
|
- //! @param testTargetExcludeList The test target exclusion list to lookup.
|
|
|
- //! @param testTargets The list of test targets to select from.
|
|
|
- //! @returns The subset of test targets in the specified list that are not on the target exclude list.
|
|
|
- template<typename TestTarget>
|
|
|
- AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>>
|
|
|
- SelectTestTargetsByExcludeList(
|
|
|
- const TestTargetExclusionList<TestTarget>& testTargetExcludeList, const AZStd::vector<const TestTarget*>& testTargets)
|
|
|
- {
|
|
|
- AZStd::vector<const TestTarget*> includedTestTargets;
|
|
|
- AZStd::vector<const TestTarget*> excludedTestTargets;
|
|
|
-
|
|
|
- if (testTargetExcludeList.IsEmpty())
|
|
|
- {
|
|
|
- return { testTargets, {} };
|
|
|
- }
|
|
|
-
|
|
|
- for (const auto& testTarget : testTargets)
|
|
|
- {
|
|
|
- if (const auto* excludedTests = testTargetExcludeList.GetExcludedTestsForTarget(testTarget);
|
|
|
- excludedTests != nullptr && excludedTests->empty())
|
|
|
- {
|
|
|
- // If the test filter is empty, the entire suite is excluded
|
|
|
- excludedTestTargets.push_back(testTarget);
|
|
|
- }
|
|
|
- else
|
|
|
- {
|
|
|
- includedTestTargets.push_back(testTarget);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return { includedTestTargets, excludedTestTargets };
|
|
|
- }
|
|
|
-
|
|
|
//! Extracts the name information from the specified test targets.
|
|
|
template<typename TestTarget>
|
|
|
AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*>& testTargets)
|
|
|
@@ -414,4 +381,179 @@ namespace TestImpact
|
|
|
AZStd::move(timedOutTests),
|
|
|
AZStd::move(unexecutedTests));
|
|
|
}
|
|
|
+
|
|
|
+ //! Selects the test targets from the specified list of test targets that are not in the specified test target exclusion list.
|
|
|
+ //! @param testTargetExcludeList The test target exclusion list to lookup.
|
|
|
+ //! @param testTargets The list of test targets to select from.
|
|
|
+ //! @returns The subset of test targets in the specified list that are not on the target exclude list.
|
|
|
+ template<typename TestTarget>
|
|
|
+ AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectTestTargetsByExcludeList(
|
|
|
+ const TestTargetExclusionList<TestTarget>& testTargetExcludeList, const AZStd::vector<const TestTarget*>& testTargets)
|
|
|
+ {
|
|
|
+ AZStd::vector<const TestTarget*> includedTestTargets;
|
|
|
+ AZStd::vector<const TestTarget*> excludedTestTargets;
|
|
|
+
|
|
|
+ if (testTargetExcludeList.IsEmpty())
|
|
|
+ {
|
|
|
+ return { testTargets, {} };
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const auto& testTarget : testTargets)
|
|
|
+ {
|
|
|
+ if (const auto* excludedTests = testTargetExcludeList.GetExcludedTestsForTarget(testTarget);
|
|
|
+ excludedTests != nullptr && excludedTests->empty())
|
|
|
+ {
|
|
|
+ // If the test filter is empty, the entire suite is excluded
|
|
|
+ excludedTestTargets.push_back(testTarget);
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ includedTestTargets.push_back(testTarget);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return { includedTestTargets, excludedTestTargets };
|
|
|
+ }
|
|
|
+
|
|
|
+ //! Utility for concatenating two vectors.
|
|
|
+ template<typename T>
|
|
|
+ static AZStd::vector<T> ConcatenateVectors(const AZStd::vector<T>& v1, const AZStd::vector<T>& v2)
|
|
|
+ {
|
|
|
+ AZStd::vector<T> result;
|
|
|
+ result.reserve(v1.size() + v2.size());
|
|
|
+ result.insert(result.end(), v1.begin(), v1.end());
|
|
|
+ result.insert(result.end(), v2.begin(), v2.end());
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ //! Utility structure for holding the pertinent data for test run reports.
|
|
|
+ template<typename TestJob>
|
|
|
+ struct TestRunData
|
|
|
+ {
|
|
|
+ TestSequenceResult m_result = TestSequenceResult::Success;
|
|
|
+ AZStd::vector<TestJob> m_jobs;
|
|
|
+ AZStd::chrono::high_resolution_clock::time_point m_relativeStartTime;
|
|
|
+ AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{ 0 };
|
|
|
+ };
|
|
|
+
|
|
|
+ //! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
|
|
|
+ //! @tparam TestRunnerFunctor The functor for running the specified tests.
|
|
|
+ //! @tparam TestJob The test engine job type returned by the functor.
|
|
|
+ //! @param maxConcurrency The maximum concurrency being used for this sequence.
|
|
|
+ //! @param policyState The policy state being used for the sequence.
|
|
|
+ //! @param suiteType The suite type used for this sequence.
|
|
|
+ //! @param timer The timer to use for the test run timings.
|
|
|
+ //! @param testRunner The test runner functor to use for each of the test runs.
|
|
|
+ //! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
|
|
|
+ //! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
|
|
|
+ //! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
|
|
|
+ //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
|
|
+ //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the
|
|
|
+ //! tests.
|
|
|
+ //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
|
|
+ //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
|
|
+ //! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
|
|
|
+ template<typename TestTarget, typename TestRunnerFunctor, typename TestJob>
|
|
|
+ Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
|
|
|
+ size_t maxConcurrency,
|
|
|
+ const ImpactAnalysisSequencePolicyState& policyState,
|
|
|
+ SuiteType suiteType,
|
|
|
+ const Timer& sequenceTimer,
|
|
|
+ const TestRunnerFunctor& testRunner,
|
|
|
+ const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
|
|
|
+ const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
|
|
|
+ const AZStd::vector<const TestTarget*>& discardedTestTargets,
|
|
|
+ const AZStd::vector<const TestTarget*>& draftedTestTargets,
|
|
|
+ const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
|
|
+ const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
|
|
+ AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
|
|
|
+ AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
|
|
|
+ AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
|
|
|
+ AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
|
|
|
+ {
|
|
|
+ TestRunData<TestJob> selectedTestRunData, draftedTestRunData;
|
|
|
+ AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
|
|
|
+
|
|
|
+ // Extract the client facing representation of selected, discarded and drafted test targets
|
|
|
+ const Client::TestRunSelection selectedTests(
|
|
|
+ ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
|
|
|
+ const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
|
|
|
+ const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
|
|
|
+
|
|
|
+ // Inform the client that the sequence is about to start
|
|
|
+ if (testSequenceStartCallback.has_value())
|
|
|
+ {
|
|
|
+ (*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
|
|
|
+ }
|
|
|
+
|
|
|
+ // We share the test run complete handler between the selected and drafted test runs as to present them together as one
|
|
|
+ // continuous test sequence to the client rather than two discrete test runs
|
|
|
+ const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
|
|
|
+ TestRunCompleteCallbackHandler<TestTarget> testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
|
|
|
+
|
|
|
+ const auto gatherTestRunData = [&sequenceTimer, &testRunner, &testRunCompleteHandler, &globalTimeout](
|
|
|
+ const AZStd::vector<const TestTarget*>& testsTargets, TestRunData<TestJob>& testRunData)
|
|
|
+ {
|
|
|
+ const Timer testRunTimer;
|
|
|
+ testRunData.m_relativeStartTime = testRunTimer.GetStartTimePointRelative(sequenceTimer);
|
|
|
+ auto [result, jobs] = testRunner(testsTargets, testRunCompleteHandler, globalTimeout);
|
|
|
+ testRunData.m_result = result;
|
|
|
+ testRunData.m_jobs = AZStd::move(jobs);
|
|
|
+ testRunData.m_duration = testRunTimer.GetElapsedMs();
|
|
|
+ };
|
|
|
+
|
|
|
+ if (!includedSelectedTestTargets.empty())
|
|
|
+ {
|
|
|
+ // Run the selected test targets and collect the test run results
|
|
|
+ gatherTestRunData(includedSelectedTestTargets, selectedTestRunData);
|
|
|
+
|
|
|
+ // Carry the remaining global sequence time over to the drafted test run
|
|
|
+ if (globalTimeout.has_value())
|
|
|
+ {
|
|
|
+ const auto elapsed = selectedTestRunData.m_duration;
|
|
|
+ sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!draftedTestTargets.empty())
|
|
|
+ {
|
|
|
+ // Run the drafted test targets and collect the test run results
|
|
|
+ gatherTestRunData(draftedTestTargets, draftedTestRunData);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Generate the sequence report for the client
|
|
|
+ const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
|
|
|
+ maxConcurrency,
|
|
|
+ testTargetTimeout,
|
|
|
+ globalTimeout,
|
|
|
+ policyState,
|
|
|
+ suiteType,
|
|
|
+ selectedTests,
|
|
|
+ discardedTests,
|
|
|
+ draftedTests,
|
|
|
+ GenerateTestRunReport(
|
|
|
+ selectedTestRunData.m_result,
|
|
|
+ selectedTestRunData.m_relativeStartTime,
|
|
|
+ selectedTestRunData.m_duration,
|
|
|
+ selectedTestRunData.m_jobs),
|
|
|
+ GenerateTestRunReport(
|
|
|
+ draftedTestRunData.m_result,
|
|
|
+ draftedTestRunData.m_relativeStartTime,
|
|
|
+ draftedTestRunData.m_duration,
|
|
|
+ draftedTestRunData.m_jobs));
|
|
|
+
|
|
|
+ // Inform the client that the sequence has ended
|
|
|
+ if (testSequenceEndCallback.has_value())
|
|
|
+ {
|
|
|
+ (*testSequenceEndCallback)(sequenceReport);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Update the dynamic dependency map with the latest coverage data (if any)
|
|
|
+ if (updateCoverage.has_value())
|
|
|
+ {
|
|
|
+ (*updateCoverage)(ConcatenateVectors(selectedTestRunData.m_jobs, draftedTestRunData.m_jobs));
|
|
|
+ }
|
|
|
+
|
|
|
+ return sequenceReport;
|
|
|
+ }
|
|
|
} // namespace TestImpact
|
John