|
|
@@ -12,6 +12,7 @@
|
|
|
|
|
|
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
|
|
|
#include <AzToolsFramework/Asset/AssetProcessorMessages.h>
|
|
|
+#include <AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h>
|
|
|
#include <AzToolsFramework/ToolsFileUtils/ToolsFileUtils.h>
|
|
|
|
|
|
#include <AzTest/AzTest.h>
|
|
|
@@ -846,6 +847,325 @@ TEST_F(AssetProcessorIntermediateAssetTests, IntermediateAsset_SourceNoLongerEmi
|
|
|
|
|
|
}
|
|
|
|
|
|
+// Used for tests that work with source dependencies and intermediate assets.
|
|
|
+class AssetProcessorIntermediateAssetSourceDependencyTests
|
|
|
+ : public AssetProcessorIntermediateAssetTests
|
|
|
+{
|
|
|
+public:
|
|
|
+
|
|
|
+ // Helper function - sets up the paths to files used by this test.
|
|
|
+ void GenerateAssetPaths()
|
|
|
+ {
|
|
|
+ // First, prep the paths and file names in use for the test.
|
|
|
+ m_firstFileName = AZ::IO::Path(m_firstFileNameNoExtension.c_str()).ReplaceExtension(m_firstFileExtension.c_str());
|
|
|
+ m_firstFilePath = AZ::IO::Path(m_scanfolder.m_scanFolder).Append(m_firstFileName);
|
|
|
+
|
|
|
+ m_secondFileName = AZ::IO::Path(m_secondFileNameNoExtension.c_str()).ReplaceExtension(m_secondFileExtension.c_str());
|
|
|
+ m_secondFilePath = AZ::IO::Path(m_scanfolder.m_scanFolder).Append(m_secondFileName);
|
|
|
+
|
|
|
+ m_intermediateFileName = AZ::IO::Path(m_firstFileNameNoExtension.c_str()).ReplaceExtension(m_intermediateExtension.c_str());
|
|
|
+ m_intermediateAssetPath = MakePath(m_intermediateFileName.c_str(), true);
|
|
|
+ m_firstProductPath = MakePath(AZ::IO::Path(m_firstFileNameNoExtension.c_str()).ReplaceExtension(m_firstProductExtension.c_str()).c_str(), false);
|
|
|
+
|
|
|
+ // Store the path to the product asset, so that the test can examine the contents of the product asset.
|
|
|
+ m_secondProductPath =
|
|
|
+ MakePath(AZ::IO::Path(m_secondFileNameNoExtension.c_str()).ReplaceExtension(m_SecondProductExtension.c_str()).c_str(), false);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Helper function - generates the initial files ued by this test.
|
|
|
+ void CreateTestAssets()
|
|
|
+ {
|
|
|
+ AZ::Utils::WriteFile("unit test file", m_firstFilePath.c_str());
|
|
|
+ AZ::Utils::WriteFile("unit test file", m_secondFilePath.c_str());
|
|
|
+ }
|
|
|
+
|
|
|
+ // Creates three builders:
|
|
|
+ // Source A - Outputs Intermediate A
|
|
|
+ // Intermediate A - Outputs Product A
|
|
|
+ // Source B - Outputs Product B, has a source dependency on Intermediate A.
|
|
|
+ void CreateBuilders()
|
|
|
+ {
|
|
|
+ using namespace AssetBuilderSDK;
|
|
|
+
|
|
|
+ // Source A's builder, this outputs the Intermediate asset.
|
|
|
+ CreateBuilder(
|
|
|
+ m_firstFileExtension.c_str(),
|
|
|
+ MakeWildcardForExtension(m_firstFileExtension).c_str(),
|
|
|
+ m_intermediateExtension.c_str(),
|
|
|
+ true,
|
|
|
+ ProductOutputFlags::IntermediateAsset);
|
|
|
+ // Intermediate A's builder, this outputs a product asset.
|
|
|
+ CreateBuilder(
|
|
|
+ m_intermediateExtension.c_str(),
|
|
|
+ MakeWildcardForExtension(m_intermediateExtension).c_str(),
|
|
|
+ m_firstProductExtension.c_str(),
|
|
|
+ false,
|
|
|
+ ProductOutputFlags::ProductAsset);
|
|
|
+
|
|
|
+ // Source B's builder. This builder emits the intermediate A as a dependency.
|
|
|
+ m_builderInfoHandler.CreateBuilderDesc(
|
|
|
+ QString(m_secondFileExtension.c_str()),
|
|
|
+ AZ::Uuid::CreateRandom().ToFixedString().c_str(),
|
|
|
+ { AssetBuilderPattern{ MakeWildcardForExtension(m_secondFileExtension), AssetBuilderPattern::Wildcard } },
|
|
|
+ [this]([[maybe_unused]] const CreateJobsRequest& request, CreateJobsResponse& response)
|
|
|
+ {
|
|
|
+ for (const auto& platform : request.m_enabledPlatforms)
|
|
|
+ {
|
|
|
+ response.m_createJobOutputs.push_back(
|
|
|
+ JobDescriptor{ "fingerprint", "Source B - Product", platform.m_identifier.c_str() });
|
|
|
+
|
|
|
+ // Create the dependency on the path to the intermediate asset.
|
|
|
+ response.m_createJobOutputs.back().m_jobDependencyList.push_back(JobDependency(
|
|
|
+ m_intermediateExtension.c_str(),
|
|
|
+ platform.m_identifier,
|
|
|
+ JobDependencyType::Order,
|
|
|
+ { m_intermediateAssetPath.c_str(),
|
|
|
+ AZ::Uuid::CreateNull(),
|
|
|
+ AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Absolute }));
|
|
|
+ }
|
|
|
+ response.m_result = CreateJobsResultCode::Success;
|
|
|
+ },
|
|
|
+ [this](const ProcessJobRequest& request, ProcessJobResponse& response)
|
|
|
+ {
|
|
|
+ AZ::IO::Path outputFile = request.m_sourceFile;
|
|
|
+
|
|
|
+ outputFile.ReplaceExtension(m_SecondProductExtension.c_str());
|
|
|
+ outputFile = AZ::IO::Path(request.m_tempDirPath).Append(outputFile);
|
|
|
+
|
|
|
+ // Write if the intermediate exists or not to the product asset.
|
|
|
+ bool intermediateProductExists = (AZ::IO::FileIOBase::GetInstance()->Exists(m_firstProductPath.c_str()));
|
|
|
+
|
|
|
+ AZStd::string toWrite = m_intermediateProductExistsString;
|
|
|
+ if (!intermediateProductExists)
|
|
|
+ {
|
|
|
+ toWrite = m_intermediateProductDoesNotExistString;
|
|
|
+ }
|
|
|
+ AZ::Utils::WriteFile(toWrite.c_str(), outputFile.c_str());
|
|
|
+
|
|
|
+ auto product =
|
|
|
+ JobProduct{ outputFile.c_str(), AZ::Data::AssetType::CreateName(m_SecondProductExtension.c_str()), AssetSubId };
|
|
|
+
|
|
|
+ product.m_outputFlags = ProductOutputFlags::ProductAsset;
|
|
|
+
|
|
|
+ product.m_dependenciesHandled = true;
|
|
|
+ response.m_outputProducts.push_back(product);
|
|
|
+ response.m_resultCode = ProcessJobResult_Success;
|
|
|
+ },
|
|
|
+ "fingerprint");
|
|
|
+ }
|
|
|
+
|
|
|
+ void SetUp() override
|
|
|
+ {
|
|
|
+ AssetProcessorIntermediateAssetTests::SetUp();
|
|
|
+
|
|
|
+ GenerateAssetPaths();
|
|
|
+ CreateTestAssets();
|
|
|
+
|
|
|
+ // Jobs with dependencies need those dependencies to have updated the asset catalog before the job with the dependency runs.
|
|
|
+ SetCatalogToUpdateOnJobCompletion();
|
|
|
+
|
|
|
+ CreateBuilders();
|
|
|
+ }
|
|
|
+
|
|
|
+ void TearDown() override
|
|
|
+ {
|
|
|
+ AssetProcessorIntermediateAssetTests::TearDown();
|
|
|
+ }
|
|
|
+
|
|
|
+protected:
|
|
|
+
|
|
|
+ AZStd::string MakeWildcardForExtension(const AZStd::string& extension)
|
|
|
+ {
|
|
|
+ return AZStd::string::format("*.%.*s", AZ_STRING_ARG(extension));
|
|
|
+ }
|
|
|
+
|
|
|
+ AZStd::string m_firstFileExtension = "a_source";
|
|
|
+ AZStd::string m_firstFileNameNoExtension = "firstfile";
|
|
|
+ AZ::IO::Path m_firstFileName;
|
|
|
+ AZ::IO::Path m_firstFilePath;
|
|
|
+
|
|
|
+ AZStd::string m_intermediateExtension = "a_intermediate";
|
|
|
+ AZ::IO::Path m_intermediateFileName;
|
|
|
+ AZ::IO::Path m_intermediateAssetPath;
|
|
|
+
|
|
|
+ AZStd::string m_firstProductExtension = "a_product";
|
|
|
+ AZStd::string m_firstProductPath;
|
|
|
+
|
|
|
+ AZStd::string m_secondFileExtension = "b_source";
|
|
|
+ AZStd::string m_secondFileNameNoExtension = "secondfile";
|
|
|
+ AZ::IO::Path m_secondFileName;
|
|
|
+ AZ::IO::Path m_secondFilePath;
|
|
|
+
|
|
|
+ AZStd::string m_SecondProductExtension = "b_product";
|
|
|
+ AZStd::string m_secondProductPath;
|
|
|
+
|
|
|
+ AZStd::string m_intermediateProductExistsString = "Intermediate product exists.";
|
|
|
+ AZStd::string m_intermediateProductDoesNotExistString = "Intermediate product does not exist.";
|
|
|
+
|
|
|
+};
|
|
|
+
|
|
|
+TEST_F(AssetProcessorIntermediateAssetSourceDependencyTests, SourceDependencyIsIntermediateAsset_NotInitiallyAvailable_JobsWaitsForIntermediateJobToExistAndRun)
|
|
|
+{
|
|
|
+ // This is a regression test for a situation where Job B depends on an intermediate asset job (Job A -> Intermediate Job A).
|
|
|
+ // Before this was fixed, Job B would be queued before Job A and Job B was unaware that Job A created Intermediate Job A.
|
|
|
+ // After the fix, jobs with missing dependencies are queued to run later than other jobs, and Common platform jobs (Job A)
|
|
|
+ // are prioritized in the queue.
|
|
|
+ // Setup:
|
|
|
+ // Builder for Source A is created. Processes "a_source" assets into "a_intermediate" assets.
|
|
|
+ // Builder for Intermediate A is created. Processes "a_intermediate" into "a_product" assets.
|
|
|
+ // Builder for Source B is created. Emits a job dependency specifically on the Intermediate A job in this test setup.
|
|
|
+ // A source file for A and B are created.
|
|
|
+ // Test:
|
|
|
+ // Jobs are both queued for Source A and Source B.
|
|
|
+ // Source A job runs first, because it does not have a missing dependency.
|
|
|
+ // Call asset processing updating in a specific order, out of processEvents order, because
|
|
|
+ // of a race condition where Intermediate A hasn't been found and had a job created by the Asset Processor
|
|
|
+ // before Job B comes up as next in the queue, causing Job B to report that it's running before its dependency is resolved.
|
|
|
+ // Instead, ScheduleNextUpdate -> ProcessFilesToExamineQueue -> CheckForIdle will make sure that the Intermediate A job is emitted.
|
|
|
+ // Verify that there are two jobs ready to submitted to the resource compiler: A new Job B with the dependency resolved, and Intermediate A.
|
|
|
+ // Process assets twice.
|
|
|
+ // Verify that the jobs run in order, and that the product of Intermediate A is available on disk during the processing of Job B.
|
|
|
+
|
|
|
+
|
|
|
+ QMetaObject::invokeMethod(
|
|
|
+ m_assetProcessorManager.get(), "AssessAddedFile", Qt::QueuedConnection, Q_ARG(QString, m_firstFilePath.c_str()));
|
|
|
+ QMetaObject::invokeMethod(
|
|
|
+ m_assetProcessorManager.get(), "AssessAddedFile", Qt::QueuedConnection, Q_ARG(QString, m_secondFilePath.c_str()));
|
|
|
+
|
|
|
+ QCoreApplication::processEvents();
|
|
|
+ m_fileCompiled = false;
|
|
|
+ m_fileFailed = false;
|
|
|
+ RunFile(2, 2);
|
|
|
+
|
|
|
+ // Queue both jobs at the same time, so that RCQueueSortModel.cpp will order these jobs and run them in order.
|
|
|
+ // TODO: Note that the PC job (asset B) will always run before the Common job (asset A) because the system
|
|
|
+ // purposely prioritizes host platform jobs first.
|
|
|
+ ASSERT_EQ(m_jobDetailsList.size(), 2);
|
|
|
+ // One of the two jobs should have the missing source dependency flagged for follow up.
|
|
|
+ EXPECT_NE(m_jobDetailsList[0].m_hasMissingSourceDependency, m_jobDetailsList[1].m_hasMissingSourceDependency);
|
|
|
+ m_rc->JobSubmitted(m_jobDetailsList[0]);
|
|
|
+ m_rc->JobSubmitted(m_jobDetailsList[1]);
|
|
|
+ m_jobDetailsList.clear();
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform("pc"), 1);
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform(AssetBuilderSDK::CommonPlatformName), 1);
|
|
|
+
|
|
|
+ UnitTests::JobSignalReceiver receiver;
|
|
|
+
|
|
|
+ // Process the first asset in the queue, which will be the job for A, because B had a missing dependency and was made lower priority.
|
|
|
+ m_rc->DispatchJobsImpl();
|
|
|
+
|
|
|
+ // Pause dispatching. In this test scenario, with only two assets queued,
|
|
|
+ // it's likely that Job B will finish before Job B is re-issued due to the newly
|
|
|
+ // updated source asset.
|
|
|
+ // This happens frequently when step-through debugging this function with breakpoints.
|
|
|
+ // This can also happen when running many tests in parallel: There are a lot of async calls
|
|
|
+ // in this test, and depending on the state of the machine running this test, those may resolve
|
|
|
+ // in a different order or timing. Pausing dispatching until the moment that dispatchJobsImpl is called
|
|
|
+ // mitigates this: Jobs only execute when this test needs them to.
|
|
|
+ m_rc->SetDispatchPaused(true);
|
|
|
+
|
|
|
+ receiver.WaitForFinish();
|
|
|
+
|
|
|
+ QCoreApplication::processEvents(); // RCJob::Finished : Once more to trigger the JobFinished event
|
|
|
+ QCoreApplication::processEvents(); // RCController::FinishJob : Again to trigger the Finished event
|
|
|
+
|
|
|
+ // Product B shouldn't exist yet because A was processed first.
|
|
|
+ EXPECT_FALSE(AZ::IO::FileIOBase::GetInstance()->Exists(m_secondProductPath.c_str()));
|
|
|
+
|
|
|
+ // Emit that A was finished processing.
|
|
|
+ m_assetProcessorManager->AssetProcessed(m_processedJobEntry, m_processJobResponse);
|
|
|
+
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform("pc"), 1);
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform(AssetBuilderSDK::CommonPlatformName), 0);
|
|
|
+
|
|
|
+ // Manually call each step to make sure the newly created intermediate asset gets discovered and queued
|
|
|
+ // before the pending job with a missing dependency is run, because the queue has one entry right now.
|
|
|
+ // Otherwise, it's likely the resource compiler will pick up the next job before the intermediate job is processed.
|
|
|
+ // Which means the missing dependency warning will fire off and cause this test to fail.
|
|
|
+ // This is a race condition in asset processing: If the last asset to process fills the missing dependency of the
|
|
|
+ // next job in the queue, then there's a brief period where intermediate asset hasn't been discovered yet and isn't queued,
|
|
|
+ // so the next job, with the missing dependency will run. This is mitigated by having the Common platform jobs run
|
|
|
+ // before non-Common platform jobs. Additional mitigation isn't currently necessary.
|
|
|
+ m_assetProcessorManager->ScheduleNextUpdate();
|
|
|
+ m_assetProcessorManager->ProcessFilesToExamineQueue();
|
|
|
+ // Call CheckForIdle twice, once for each pending asset.
|
|
|
+ m_assetProcessorManager->CheckForIdle();
|
|
|
+ m_assetProcessorManager->CheckForIdle();
|
|
|
+
|
|
|
+ // Make sure events are dispatched and m_jobDetailsList is updated with both jobs.
|
|
|
+
|
|
|
+ // The queue should now be the job to process the intermediate asset, and the re-created Job B, which no longer has a missing dependency.
|
|
|
+ ASSERT_EQ(m_jobDetailsList.size(), 2);
|
|
|
+ // Neither job in the queue should have the missing source dependency flag.
|
|
|
+ EXPECT_FALSE(m_jobDetailsList[0].m_hasMissingSourceDependency);
|
|
|
+ EXPECT_FALSE(m_jobDetailsList[1].m_hasMissingSourceDependency);
|
|
|
+ m_rc->JobSubmitted(m_jobDetailsList[0]);
|
|
|
+ m_rc->JobSubmitted(m_jobDetailsList[1]);
|
|
|
+ m_jobDetailsList.clear();
|
|
|
+
|
|
|
+ // The platform is for the target output, not for the platform the source was on, so the intermediate asset job will
|
|
|
+ // have the PC platform.
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform("pc"), 2);
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform(AssetBuilderSDK::CommonPlatformName), 0);
|
|
|
+
|
|
|
+ // The original Job B, with the missing dependency, was canceled.
|
|
|
+ // Verify the pending job list matches the model's queue.
|
|
|
+ // Canceling jobs doesn't guarantee the pending count will be updated, but it does guarantee the row count
|
|
|
+ // of the model is updated. This check verifies that the call to cancel the job was done correctly, and updated
|
|
|
+ // the pending list.
|
|
|
+ AssetProcessor::RCQueueSortModel& sortModel = m_rc->GetRCQueueSortModel();
|
|
|
+ EXPECT_EQ(sortModel.rowCount(), m_rc->NumberOfPendingJobsPerPlatform("pc"));
|
|
|
+ m_rc->SetDispatchPaused(false);
|
|
|
+ m_rc->DispatchJobsImpl();
|
|
|
+ m_rc->SetDispatchPaused(true);
|
|
|
+
|
|
|
+ receiver.WaitForFinish();
|
|
|
+
|
|
|
+ QCoreApplication::processEvents(); // RCJob::Finished : Once more to trigger the JobFinished event
|
|
|
+ QCoreApplication::processEvents(); // RCController::FinishJob : Again to trigger the Finished event
|
|
|
+
|
|
|
+ // Mark this asset as processed, so AP will move on to the next steps.
|
|
|
+ m_assetProcessorManager->AssetProcessed(m_processedJobEntry, m_processJobResponse);
|
|
|
+
|
|
|
+ // Need to call process events multiple times, to get the job details list populated with the intermediate asset job
|
|
|
+ // Due to timing of this test, these events may end up running in different process events calls.
|
|
|
+ // updates a lot of general AssetProcessor systems:
|
|
|
+ // AssetProcessorManager::ScheduleNextUpdate, AssetProcessorManager::ProcessFilesToExamineQueue,
|
|
|
+ // AssetProcessorManager::QueueIdleCheck, AssetProcessorManager::ProcessBuilders, and more.
|
|
|
+ // Also updates several AssetProcessor systems that the previous step updates,
|
|
|
+ // but the main events this is run for is two calls to AssetProcessorManager::AssetToProcess
|
|
|
+ // which puts Cache/Intermediate Assets/firstfile.a_intermediate and secondfile.b_source in m_jobDetailsList
|
|
|
+ // Make sure the job requests are populated and ready to go, without this, RCJob::PopulateProcessJobRequest sometimes crashes accessing job info.
|
|
|
+ QCoreApplication::processEvents();
|
|
|
+ QCoreApplication::processEvents();
|
|
|
+ QCoreApplication::processEvents();
|
|
|
+
|
|
|
+ // Verify that the job for the second asset didn't process yet, because it has a job dependency on the intermediate asset job.
|
|
|
+ EXPECT_FALSE(AZ::IO::FileIOBase::GetInstance()->Exists(m_secondProductPath.c_str()));
|
|
|
+
|
|
|
+ // Emit that the intermediate job was finished processing.
|
|
|
+ m_assetProcessorManager->AssetProcessed(m_processedJobEntry, m_processJobResponse);
|
|
|
+
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform("pc"), 1);
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform(AssetBuilderSDK::CommonPlatformName), 0);
|
|
|
+
|
|
|
+ // Make sure all remaning non-canceled jobs are processed.
|
|
|
+ m_rc->SetDispatchPaused(false);
|
|
|
+ WaitForNextJobToProcess(receiver);
|
|
|
+
|
|
|
+ // Verify the final product is marked as existing.
|
|
|
+ auto readResult = AZ::Utils::ReadFile<AZStd::string>(m_secondProductPath.c_str(), AZStd::numeric_limits<size_t>::max());
|
|
|
+ EXPECT_TRUE(readResult.IsSuccess());
|
|
|
+ EXPECT_EQ(readResult.GetValue().compare(m_intermediateProductExistsString), 0);
|
|
|
+
|
|
|
+ // Examine the queue directly : There should be nothing left in the pending job queue, or the sort model's row count.
|
|
|
+ EXPECT_EQ(m_rc->NumberOfPendingJobsPerPlatform("pc"), 0);
|
|
|
+ EXPECT_EQ(sortModel.GetNextPendingJob(), nullptr);
|
|
|
+
|
|
|
+ // The canceled job was removed from the actual sort model, it just wasn't removed from the pending per platform list.
|
|
|
+ EXPECT_EQ(sortModel.rowCount(), 0);
|
|
|
+}
|
|
|
+
|
|
|
TEST_F(AssetProcessorManagerTest, WarningsAndErrorsReported_SuccessfullySavedToDatabase)
|
|
|
{
|
|
|
// This tests the JobDiagnosticTracker: Warnings/errors reported to it should be recorded in the database when AssetProcessed is fired and able to be retrieved when querying job status
|
AMZN-stankowi