/* * Copyright (c) Contributors to the Open 3D Engine Project. * For complete copyright and license terms please see the LICENSE at the root of this distribution. * * SPDX-License-Identifier: Apache-2.0 OR MIT * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace UnitTest { namespace { bool CreateDummyFile(const QString& fullPathToFile, const QString& tempStr = {}) { QFileInfo fi(fullPathToFile); QDir fp(fi.path()); fp.mkpath("."); QFile writer(fullPathToFile); if (!writer.open(QFile::ReadWrite)) { return false; } { QTextStream stream(&writer); stream << tempStr << Qt::endl; } writer.close(); return true; } class ArchiveComponentTest : public UnitTest::LeakDetectionFixture { public: QStringList CreateArchiveFileList() { QStringList returnList; returnList.append("basicfile.txt"); returnList.append("basicfile2.txt"); returnList.append("testfolder/folderfile.txt"); returnList.append("testfolder2/sharedfolderfile.txt"); returnList.append("testfolder2/sharedfolderfile2.txt"); returnList.append("testfolder3/testfolder4/depthfile.bat"); return returnList; } QString GetArchiveFolderName() { return "archive"; } QString GetExtractFolderName() { return "extracted"; } void CreateArchiveFolder(QString archiveFolderName, QStringList fileList) { QDir tempPath = QDir(m_tempDir.GetDirectory()).filePath(archiveFolderName); for (const auto& thisFile : fileList) { QString absoluteTestFilePath = tempPath.absoluteFilePath(thisFile); EXPECT_TRUE(CreateDummyFile(absoluteTestFilePath)); } } QString CreateArchiveListTextFile() { QString listFilePath = QDir(m_tempDir.GetDirectory()).absoluteFilePath("filelist.txt"); QString textContent = CreateArchiveFileList().join("\n"); EXPECT_TRUE(CreateDummyFile(listFilePath, textContent)); return listFilePath; } void CreateArchiveFolder() { CreateArchiveFolder(GetArchiveFolderName(), CreateArchiveFileList()); } QString GetArchivePath() { return QDir(m_tempDir.GetDirectory()).filePath("TestArchive.pak"); } QString GetArchiveFolder() { return QDir(m_tempDir.GetDirectory()).filePath(GetArchiveFolderName()); } QString GetExtractFolder() { return QDir(m_tempDir.GetDirectory()).filePath(GetExtractFolderName()); } bool CreateArchive() { std::future createResult; AzToolsFramework::ArchiveCommandsBus::BroadcastResult(createResult, &AzToolsFramework::ArchiveCommandsBus::Events::CreateArchive, GetArchivePath().toUtf8().constData(), GetArchiveFolder().toUtf8().constData()); bool result = createResult.get(); return result; } void SetUp() override { m_app.reset(aznew ToolsTestApplication("ArchiveComponentTest")); AZ::ComponentApplication::StartupParameters startupParameters; startupParameters.m_loadSettingsRegistry = false; m_app->Start(AzFramework::Application::Descriptor(), startupParameters); // Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is // shared across the whole engine, if multiple tests are run in parallel, the saving could cause a crash // in the unit tests. AZ::UserSettingsComponentRequestBus::Broadcast(&AZ::UserSettingsComponentRequests::DisableSaveOnFinalize); if (auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); fileIoBase != nullptr) { QDir cacheFolder(m_tempDir.GetDirectory()); // set the product tree folder to somewhere besides the root temp dir. // This is to avoid error spam - if you try to write to the Cache folder or a subfolder, // AZ::IO will issue an error, since the cache is supposed to be read-only. // here we set it to (tempFolder)/Cache subfolder so that if you want a folder in your test to act like // the read-only cache folder, you can use that folder, but otherwise, all other folders are fair game to // use for your tests without triggering the "you cannot write to the cache" error. fileIoBase->SetAlias("@products@", cacheFolder.absoluteFilePath("Cache").toUtf8().constData()); } } void TearDown() override { m_app->Stop(); m_app.reset(); } AZStd::unique_ptr m_app; AZ::Test::ScopedAutoTempDirectory m_tempDir; }; TEST_F(ArchiveComponentTest, CreateArchive_FilesAtThreeDepths_ArchiveCreated) { CreateArchiveFolder(); bool createResult = CreateArchive(); EXPECT_TRUE(createResult); } TEST_F(ArchiveComponentTest, ListFilesInArchive_FilesAtThreeDepths_FilesFound) { CreateArchiveFolder(); EXPECT_EQ(CreateArchive(), true); AZStd::vector fileList; bool listResult{ false }; AzToolsFramework::ArchiveCommandsBus::BroadcastResult(listResult, &AzToolsFramework::ArchiveCommandsBus::Events::ListFilesInArchive, GetArchivePath().toUtf8().constData(), fileList); EXPECT_TRUE(listResult); EXPECT_EQ(fileList.size(), 6); } TEST_F(ArchiveComponentTest, CreateDeltaCatalog_AssetsNotRegistered_Failure) { QStringList fileList = CreateArchiveFileList(); CreateArchiveFolder(GetArchiveFolderName(), fileList); bool createResult = CreateArchive(); EXPECT_EQ(createResult, true); bool catalogCreated{ true }; AZ::Test::AssertAbsorber assertAbsorber; AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toUtf8().constData(), true); EXPECT_EQ(catalogCreated, false); } TEST_F(ArchiveComponentTest, AddFilesToArchive_FromListFile_Success) { QString listFile = CreateArchiveListTextFile(); CreateArchiveFolder(GetArchiveFolderName(), CreateArchiveFileList()); std::future addResult; AzToolsFramework::ArchiveCommandsBus::BroadcastResult( addResult, &AzToolsFramework::ArchiveCommandsBus::Events::AddFilesToArchive, GetArchivePath().toUtf8().constData(), GetArchiveFolder().toUtf8().constData(), listFile.toUtf8().constData()); bool result = addResult.get(); EXPECT_TRUE(result); } TEST_F(ArchiveComponentTest, ExtractArchive_AllFiles_Success) { CreateArchiveFolder(); bool createResult = CreateArchive(); EXPECT_TRUE(createResult); std::future extractResult; AzToolsFramework::ArchiveCommandsBus::BroadcastResult( extractResult, &AzToolsFramework::ArchiveCommandsBus::Events::ExtractArchive, GetArchivePath().toUtf8().constData(), GetExtractFolder().toUtf8().constData()); bool result = extractResult.get(); EXPECT_TRUE(result); QStringList archiveFiles = CreateArchiveFileList(); for (const auto& file : archiveFiles) { QString fullFilePath = QDir(GetExtractFolder()).absoluteFilePath(file); QFileInfo fi(fullFilePath); EXPECT_TRUE(fi.exists()); } } TEST_F(ArchiveComponentTest, CreateDeltaCatalog_ArchiveWithoutCatalogAssetsRegistered_Success) { QStringList fileList = CreateArchiveFileList(); CreateArchiveFolder(GetArchiveFolderName(), fileList); bool createResult = CreateArchive(); EXPECT_EQ(createResult, true); for (const auto& thisPath : fileList) { AZ::Data::AssetInfo newInfo; newInfo.m_relativePath = thisPath.toUtf8().constData(); newInfo.m_assetType = AZ::Uuid::CreateRandom(); newInfo.m_sizeBytes = 100; // Arbitrary AZ::Data::AssetId generatedID(AZ::Uuid::CreateRandom()); newInfo.m_assetId = generatedID; AZ::Data::AssetCatalogRequestBus::Broadcast(&AZ::Data::AssetCatalogRequestBus::Events::RegisterAsset, generatedID, newInfo); } AZ_TEST_START_TRACE_SUPPRESSION; bool catalogCreated{ false }; AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toUtf8().constData(), true); AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // the above raises at least one complaint, but is os specific, since it creates a file in the cache (and then deletes it) EXPECT_EQ(catalogCreated, true); } TEST_F(ArchiveComponentTest, SUITE_periodic_ArchiveAsyncMemoryCorruptionTest) { // simulate the way the Asset Processor might create many archives asynchronously, overlapping. // The general pattern the AP uses is that NCPUs threads are created, and each thread could be creating an archive // at the same time. Each thread is operating on its own temp directory, and calls two APIs: // CreateArchive (every time), and then AddFileToArchive (some of the time). // There is always a file in the archive, but not always one in the extra API call. // to simulate this, we're going to start 8 threads // those 8 threads will continuously create files in a folder, then archive them, then add additional files to that archive. const int numThreads = 8; const int numIterationsPerThread = 100; // takes about 20sec in debug on good HW with ASAN, much faster in profile. auto threadFn = [this](int threadIndex, int iterations) { const int numDummyFiles = 5; for (int iteration = 0; iteration < iterations; ++iteration) { // create a temp folder and then 5 dummy files in that folder to represent the files that will be archived // tempfolder/archive_n_n = folder containing files to archive initially, in the "CreateArchive" API call. // tempfolder/extra_n_n = folder containing files to add to archive afterwards, in the "AddFilesToArchive" API call. // tempfolder/TestArchive_n_n.zip = archive output file. // tempfolder/extra_n_n/filelist.txt = list of files to add to archive in the "AddFilesToArchive" call. // we do not attempt to read the archive back, this is just a thrash test. QString folderName = QString("archive%1_%2").arg(threadIndex).arg(iteration); QString extraFolderName = QString("extra%1_%2").arg(threadIndex).arg(iteration); QString archivePath = QDir(m_tempDir.GetDirectory()).filePath(QString("TestArchive%1_%2.zip").arg(threadIndex).arg(iteration)); QString folderPath = QDir(m_tempDir.GetDirectory()).filePath(folderName); QString extraFolderPath = QDir(m_tempDir.GetDirectory()).filePath(extraFolderName); QString dummyFileContent; for (int fileToArchive = 0; fileToArchive < numDummyFiles; ++fileToArchive) { QString filePath = QDir(folderPath).filePath(QString("file%1.txt").arg(fileToArchive)); QString extraFileName = QString("extrafile%1.txt").arg(fileToArchive); QString extraFilePath = QDir(extraFolderPath).filePath(extraFileName); CreateDummyFile(filePath, QString(1024 * iteration, QChar('C'))); CreateDummyFile(extraFilePath, QString(1024 * iteration, QChar('C'))); dummyFileContent.append(QDir::toNativeSeparators(extraFileName)); dummyFileContent.append("\n"); } QString fileListPath = QDir(extraFolderPath).filePath("filelist.txt"); CreateDummyFile(fileListPath, dummyFileContent); std::future createResult; AzToolsFramework::ArchiveCommandsBus::BroadcastResult( createResult, &AzToolsFramework::ArchiveCommandsBus::Events::CreateArchive, archivePath.toUtf8().constData(), folderPath.toUtf8().constData()); EXPECT_TRUE(createResult.valid() ? createResult.get() : false); std::future addResult; AzToolsFramework::ArchiveCommandsBus::BroadcastResult( addResult, &AzToolsFramework::ArchiveCommandsBus::Events::AddFilesToArchive, archivePath.toUtf8().constData(), extraFolderPath.toUtf8().constData(), fileListPath.toUtf8().constData()); EXPECT_TRUE(addResult.valid() ? addResult.get() : false); } }; // spawn 8 threads to do the above and then wait for all of them to complete. AZStd::vector threads; for (int i = 0; i < numThreads; ++i) { threads.emplace_back(threadFn, i, numIterationsPerThread); } for (int i = 0; i < numThreads; ++i) { if (threads[i].joinable()) { threads[i].join(); } } } } }