AssetCatalog.cpp 80 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884
  1. /*
  2. * Copyright (c) Contributors to the Open 3D Engine Project.
  3. * For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. *
  5. * SPDX-License-Identifier: Apache-2.0 OR MIT
  6. *
  7. */
  8. #include "native/AssetManager/AssetCatalog.h"
  9. #include <AzCore/Asset/AssetSerializer.h>
  10. #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
  11. #include <AzCore/std/string/wildcard.h>
  12. #include <AzFramework/API/ApplicationAPI.h>
  13. #include <AzFramework/FileTag/FileTagBus.h>
  14. #include <AzFramework/FileTag/FileTag.h>
  15. #include <AzToolsFramework/API/AssetDatabaseBus.h>
  16. #include <QElapsedTimer>
  17. #include "PathDependencyManager.h"
  18. #include <utilities/UuidManager.h>
  19. namespace AssetProcessor
  20. {
  21. AssetCatalog::AssetCatalog(QObject* parent, AssetProcessor::PlatformConfiguration* platformConfiguration)
  22. : QObject(parent)
  23. , m_platformConfig(platformConfiguration)
  24. , m_registryBuiltOnce(false)
  25. , m_registriesMutex(QMutex::Recursive)
  26. {
  27. for (const AssetBuilderSDK::PlatformInfo& info : m_platformConfig->GetEnabledPlatforms())
  28. {
  29. if (info.m_identifier == AssetBuilderSDK::CommonPlatformName)
  30. {
  31. // Currently the Common platform is not supported as a product asset platform
  32. continue;
  33. }
  34. m_platforms.push_back(QString::fromUtf8(info.m_identifier.c_str()));
  35. }
  36. [[maybe_unused]] bool computedCacheRoot = AssetUtilities::ComputeProjectCacheRoot(m_cacheRoot);
  37. AZ_Assert(computedCacheRoot, "Could not compute cache root for AssetCatalog");
  38. // save 30mb for this. Really large projects do get this big (and bigger)
  39. // if you don't do this, things get fragmented very fast.
  40. m_saveBuffer.reserve(1024 * 1024 * 30);
  41. AssetUtilities::ComputeProjectPath();
  42. if (!ConnectToDatabase())
  43. {
  44. AZ_Error("AssetCatalog", false, "Failed to connect to sqlite database");
  45. }
  46. AssetRegistryRequestBus::Handler::BusConnect();
  47. AzToolsFramework::AssetSystemRequestBus::Handler::BusConnect();
  48. AzToolsFramework::ToolsAssetSystemBus::Handler::BusConnect();
  49. AZ::Data::AssetCatalogRequestBus::Handler::BusConnect();
  50. }
  51. AssetCatalog::~AssetCatalog()
  52. {
  53. AzToolsFramework::ToolsAssetSystemBus::Handler::BusDisconnect();
  54. AzToolsFramework::AssetSystemRequestBus::Handler::BusDisconnect();
  55. AssetRegistryRequestBus::Handler::BusDisconnect();
  56. AZ::Data::AssetCatalogRequestBus::Handler::BusDisconnect();
  57. SaveRegistry_Impl();
  58. }
  59. void AssetCatalog::OnAssetMessage(AzFramework::AssetSystem::AssetNotificationMessage message)
  60. {
  61. using namespace AzFramework::AssetSystem;
  62. if (message.m_type == AssetNotificationMessage::AssetChanged)
  63. {
  64. //get the full product path to determine file size
  65. AZ::Data::AssetInfo assetInfo;
  66. assetInfo.m_assetId = message.m_assetId;
  67. assetInfo.m_assetType = message.m_assetType;
  68. assetInfo.m_relativePath = message.m_data.c_str();
  69. assetInfo.m_sizeBytes = message.m_sizeBytes;
  70. QString assetPlatform{ QString::fromUtf8(message.m_platform.c_str()) };
  71. AZ_Assert(assetInfo.m_assetId.IsValid(), "AssetID is not valid!!!");
  72. AZ_Assert(!assetInfo.m_relativePath.empty(), "Product path is empty");
  73. AZ_Assert(!assetPlatform.isEmpty(), "Product platform is empty");
  74. m_catalogIsDirty = true;
  75. {
  76. QMutexLocker locker(&m_registriesMutex);
  77. m_registries[message.m_platform.c_str()].RegisterAsset(assetInfo.m_assetId, assetInfo);
  78. for (const AZ::Data::AssetId& mapping : message.m_legacyAssetIds)
  79. {
  80. if (mapping != assetInfo.m_assetId)
  81. {
  82. m_registries[assetPlatform].RegisterLegacyAssetMapping(mapping, assetInfo.m_assetId);
  83. }
  84. }
  85. m_registries[assetPlatform].SetAssetDependencies(message.m_assetId, message.m_dependencies);
  86. using namespace AzFramework::FileTag;
  87. // We are checking preload Dependency only for runtime assets
  88. AZStd::vector<AZStd::string> excludedTagsList = { FileTags[static_cast<unsigned int>(FileTagsIndex::EditorOnly)] };
  89. bool editorOnlyAsset = false;
  90. QueryFileTagsEventBus::EventResult(editorOnlyAsset, FileTagType::Exclude,
  91. &QueryFileTagsEventBus::Events::Match, message.m_data.c_str(), excludedTagsList);
  92. if (!editorOnlyAsset)
  93. {
  94. for (auto& productDependency : message.m_dependencies)
  95. {
  96. auto loadBehavior = AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(productDependency.m_flags);
  97. if (loadBehavior == AZ::Data::AssetLoadBehavior::PreLoad)
  98. {
  99. m_preloadAssetList.emplace_back(AZStd::make_pair(message.m_assetId, message.m_platform.c_str()));
  100. break;
  101. }
  102. }
  103. }
  104. }
  105. if (m_registryBuiltOnce)
  106. {
  107. Q_EMIT SendAssetMessage(message);
  108. }
  109. }
  110. else if (message.m_type == AssetNotificationMessage::AssetRemoved)
  111. {
  112. QMutexLocker locker(&m_registriesMutex);
  113. QString assetPlatform{ QString::fromUtf8(message.m_platform.c_str()) };
  114. AZ_Assert(!assetPlatform.isEmpty(), "Product platform is empty");
  115. auto found = m_registries[assetPlatform].m_assetIdToInfo.find(message.m_assetId);
  116. if (found != m_registries[assetPlatform].m_assetIdToInfo.end())
  117. {
  118. m_catalogIsDirty = true;
  119. m_registries[assetPlatform].UnregisterAsset(message.m_assetId);
  120. m_registries[assetPlatform].UnregisterLegacyAssetMappingsForAsset(message.m_assetId);
  121. if (m_registryBuiltOnce)
  122. {
  123. Q_EMIT SendAssetMessage(message);
  124. }
  125. }
  126. }
  127. }
  128. bool AssetCatalog::CheckValidatedAssets(AZ::Data::AssetId assetId, const QString& platform)
  129. {
  130. auto found = m_cachedNoPreloadDependenyAssetList.equal_range(assetId);
  131. for (auto platformIter = found.first; platformIter != found.second; ++platformIter)
  132. {
  133. if (platformIter->second == platform)
  134. {
  135. // we have already verified this asset for this run and it does not have any preload dependency for the specified platform, therefore we can safely skip it
  136. return false;
  137. }
  138. }
  139. return true;
  140. }
  141. void AssetCatalog::ValidatePreLoadDependency()
  142. {
  143. if (m_currentlyValidatingPreloadDependency)
  144. {
  145. return;
  146. }
  147. m_currentlyValidatingPreloadDependency = true;
  148. for (auto iter = m_preloadAssetList.begin(); iter != m_preloadAssetList.end(); iter++)
  149. {
  150. if (!CheckValidatedAssets(iter->first, iter->second))
  151. {
  152. continue;
  153. }
  154. AZStd::stack<AZStd::pair<AZ::Data::AssetId, AZ::Data::AssetId>> assetStack;
  155. AZStd::vector<AZ::Data::AssetId> currentAssetTree; // this is used to determine the hierarchy of asset loads.
  156. AZStd::unordered_set<AZ::Data::AssetId> currentVisitedAssetsTree;
  157. AZStd::unordered_set<AZ::Data::AssetId> allVisitedAssets;
  158. assetStack.push(AZStd::make_pair(iter->first, AZ::Data::AssetId()));
  159. bool cyclicDependencyFound = false;
  160. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  161. while (!assetStack.empty())
  162. {
  163. AZ::Data::AssetId assetId = assetStack.top().first;
  164. AZ::Data::AssetId parentAssetId = assetStack.top().second;
  165. assetStack.pop();
  166. allVisitedAssets.insert(assetId);
  167. while (currentAssetTree.size() && parentAssetId != currentAssetTree.back())
  168. {
  169. currentVisitedAssetsTree.erase(currentAssetTree.back());
  170. currentAssetTree.pop_back();
  171. };
  172. currentVisitedAssetsTree.insert(assetId);
  173. currentAssetTree.emplace_back(assetId);
  174. m_db->QueryProductDependencyBySourceGuidSubId(assetId.m_guid, assetId.m_subId, iter->second.toUtf8().constData(), [&](const AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry)
  175. {
  176. auto loadBehavior = AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(entry.m_dependencyFlags);
  177. if (loadBehavior == AZ::Data::AssetLoadBehavior::PreLoad)
  178. {
  179. AZ::Data::AssetId dependentAssetId(entry.m_dependencySourceGuid, entry.m_dependencySubID);
  180. if (currentVisitedAssetsTree.find(dependentAssetId) == currentVisitedAssetsTree.end())
  181. {
  182. if (!CheckValidatedAssets(dependentAssetId, iter->second))
  183. {
  184. // we have already verified that this asset does not have any preload dependency
  185. return true;
  186. }
  187. assetStack.push(AZStd::make_pair(dependentAssetId, assetId));
  188. }
  189. else
  190. {
  191. cyclicDependencyFound = true;
  192. AZStd::string cyclicPreloadDependencyTreeString;
  193. for (const auto& assetIdEntry : currentAssetTree)
  194. {
  195. AzToolsFramework::AssetDatabase::ProductDatabaseEntry productDatabaseEntry;
  196. m_db->GetProductBySourceGuidSubId(assetIdEntry.m_guid, assetIdEntry.m_subId, productDatabaseEntry);
  197. cyclicPreloadDependencyTreeString = cyclicPreloadDependencyTreeString + AZStd::string::format("%s ->", productDatabaseEntry.m_productName.c_str());
  198. };
  199. AzToolsFramework::AssetDatabase::ProductDatabaseEntry productDatabaseEntry;
  200. m_db->GetProductBySourceGuidSubId(dependentAssetId.m_guid, dependentAssetId.m_subId, productDatabaseEntry);
  201. cyclicPreloadDependencyTreeString = cyclicPreloadDependencyTreeString + AZStd::string::format(" %s ", productDatabaseEntry.m_productName.c_str());
  202. AzToolsFramework::AssetDatabase::ProductDatabaseEntry productDatabaseRootEntry;
  203. m_db->GetProductBySourceGuidSubId(iter->first.m_guid, iter->first.m_subId, productDatabaseRootEntry);
  204. AZ_Error(AssetProcessor::ConsoleChannel, false, "Preload circular dependency detected while processing asset (%s).\n Preload hierarchy is %s . Adjust your product dependencies for assets in this chain to break this loop.",
  205. productDatabaseRootEntry.m_productName.c_str(), cyclicPreloadDependencyTreeString.c_str());
  206. return false;
  207. }
  208. }
  209. return true;
  210. });
  211. if (cyclicDependencyFound)
  212. {
  213. currentVisitedAssetsTree.clear();
  214. currentAssetTree.clear();
  215. AZStd::stack<AZStd::pair<AZ::Data::AssetId, AZ::Data::AssetId>> emptyAssetStack;
  216. assetStack.swap(emptyAssetStack);
  217. }
  218. };
  219. if (!cyclicDependencyFound)
  220. {
  221. for (const auto& assetId : allVisitedAssets)
  222. {
  223. m_cachedNoPreloadDependenyAssetList.emplace(AZStd::make_pair(assetId, iter->second)); // assetid, platform
  224. }
  225. }
  226. }
  227. m_preloadAssetList.clear();
  228. m_cachedNoPreloadDependenyAssetList.clear();
  229. m_currentlyValidatingPreloadDependency = false;
  230. }
  231. void AssetCatalog::SaveRegistry_Impl()
  232. {
  233. bool allCatalogsSaved = true;
  234. // note that its safe not to save the catalog if the catalog is not dirty
  235. // because the engine will be accepting updates as long as the update has a higher or equal
  236. // number to the saveId, not just equal.
  237. if (m_catalogIsDirty)
  238. {
  239. m_catalogIsDirty = false;
  240. // Reflect registry for serialization.
  241. AZ::SerializeContext* serializeContext = nullptr;
  242. AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext);
  243. AZ_Assert(serializeContext, "Unable to retrieve serialize context.");
  244. if (nullptr == serializeContext->FindClassData(AZ::AzTypeInfo<AzFramework::AssetRegistry>::Uuid()))
  245. {
  246. AzFramework::AssetRegistry::ReflectSerialize(serializeContext);
  247. }
  248. // save out a catalog for each platform
  249. for (const QString& platform : m_platforms)
  250. {
  251. // Serialize out the catalog to a memory buffer, and then dump that memory buffer to stream.
  252. QElapsedTimer timer;
  253. timer.start();
  254. m_saveBuffer.clear();
  255. // allow this to grow by up to 20mb at a time so as not to fragment.
  256. // we re-use the save buffer each time to further reduce memory load.
  257. AZ::IO::ByteContainerStream<AZStd::vector<char>> catalogFileStream(&m_saveBuffer, 1024 * 1024 * 20);
  258. // these 3 lines are what writes the entire registry to the memory stream
  259. AZ::ObjectStream* objStream = AZ::ObjectStream::Create(&catalogFileStream, *serializeContext, AZ::ObjectStream::ST_BINARY);
  260. {
  261. QMutexLocker locker(&m_registriesMutex);
  262. objStream->WriteClass(&m_registries[platform]);
  263. }
  264. objStream->Finalize();
  265. // now write the memory stream out to the temp folder
  266. QString workSpace;
  267. if (!AssetUtilities::CreateTempWorkspace(workSpace))
  268. {
  269. AZ_Warning(AssetProcessor::ConsoleChannel, false, "Failed to create a temp workspace for catalog writing\n");
  270. }
  271. else
  272. {
  273. auto settingsRegistry = AZ::SettingsRegistry::Get();
  274. AZ::SettingsRegistryInterface::FixedValueString cacheRootFolder;
  275. settingsRegistry->Get(cacheRootFolder, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
  276. QString tempRegistryFile = QString("%1/%2").arg(workSpace).arg("assetcatalog.xml.tmp");
  277. QString platformCacheDir = QString("%1/%2").arg(cacheRootFolder.c_str()).arg(platform);
  278. QString actualRegistryFile = QString("%1/%2").arg(platformCacheDir).arg("assetcatalog.xml");
  279. AZ_TracePrintf(AssetProcessor::DebugChannel, "Creating asset catalog: %s --> %s\n", tempRegistryFile.toUtf8().constData(), actualRegistryFile.toUtf8().constData());
  280. AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle;
  281. if (AZ::IO::FileIOBase::GetInstance()->Open(tempRegistryFile.toUtf8().data(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary, fileHandle))
  282. {
  283. AZ::IO::FileIOBase::GetInstance()->Write(fileHandle, m_saveBuffer.data(), m_saveBuffer.size());
  284. AZ::IO::FileIOBase::GetInstance()->Close(fileHandle);
  285. // Make sure that the destination folder of the registry file exists
  286. QDir registryDir(platformCacheDir);
  287. if (!registryDir.exists())
  288. {
  289. QString absPath = registryDir.absolutePath();
  290. [[maybe_unused]] AZ::IO::Result makeDirResult = AZ::IO::FileIOBase::GetInstance()->CreatePath(absPath.toUtf8().constData());
  291. AZ_Warning(AssetProcessor::ConsoleChannel, makeDirResult, "Failed create folder %s", platformCacheDir.toUtf8().constData());
  292. }
  293. // if we succeeded in doing this, then use "rename" to move the file over the previous copy.
  294. bool moved = AssetUtilities::MoveFileWithTimeout(tempRegistryFile, actualRegistryFile, 3);
  295. allCatalogsSaved = allCatalogsSaved && moved;
  296. // warn if it failed
  297. AZ_Warning(AssetProcessor::ConsoleChannel, moved, "Failed to move %s to %s", tempRegistryFile.toUtf8().constData(), actualRegistryFile.toUtf8().constData());
  298. if (moved)
  299. {
  300. AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Saved %s catalog containing %u assets in %fs\n", platform.toUtf8().constData(), m_registries[platform].m_assetIdToInfo.size(), timer.elapsed() / 1000.0f);
  301. }
  302. }
  303. else
  304. {
  305. AZ_Warning(AssetProcessor::ConsoleChannel, false, "Failed to create catalog file %s", tempRegistryFile.toUtf8().constData());
  306. allCatalogsSaved = false;
  307. }
  308. AZ::IO::FileIOBase::GetInstance()->DestroyPath(workSpace.toUtf8().data());
  309. }
  310. }
  311. }
  312. {
  313. // scoped to minimize the duration of this mutex lock
  314. QMutexLocker locker(&m_savingRegistryMutex);
  315. m_currentlySavingCatalog = false;
  316. RegistrySaveComplete(m_currentRegistrySaveVersion, allCatalogsSaved);
  317. AssetRegistryNotificationBus::Broadcast(&AssetRegistryNotifications::OnRegistrySaveComplete, m_currentRegistrySaveVersion, allCatalogsSaved);
  318. }
  319. }
  320. AzFramework::AssetSystem::GetUnresolvedDependencyCountsResponse AssetCatalog::HandleGetUnresolvedDependencyCountsRequest(MessageData<AzFramework::AssetSystem::GetUnresolvedDependencyCountsRequest> messageData)
  321. {
  322. AzFramework::AssetSystem::GetUnresolvedDependencyCountsResponse response;
  323. {
  324. QMutexLocker locker(&m_registriesMutex);
  325. const auto& productDependencies = m_registries[messageData.m_platform].GetAssetDependencies(messageData.m_message->m_assetId);
  326. for (const AZ::Data::ProductDependency& productDependency : productDependencies)
  327. {
  328. if (m_registries[messageData.m_platform].m_assetIdToInfo.find(productDependency.m_assetId)
  329. == m_registries[messageData.m_platform].m_assetIdToInfo.end())
  330. {
  331. ++response.m_unresolvedAssetIdReferences;
  332. }
  333. }
  334. }
  335. {
  336. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  337. m_db->QueryProductDependencyBySourceGuidSubId(messageData.m_message->m_assetId.m_guid, messageData.m_message->m_assetId.m_subId, messageData.m_platform.toUtf8().constData(), [&response](const AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry)
  338. {
  339. if (!entry.m_unresolvedPath.empty() && entry.m_unresolvedPath.find('*') == entry.m_unresolvedPath.npos
  340. && !entry.m_unresolvedPath.starts_with(ExcludedDependenciesSymbol))
  341. {
  342. ++response.m_unresolvedPathReferences;
  343. }
  344. return true;
  345. });
  346. }
  347. return response;
  348. }
  349. void AssetCatalog::HandleSaveAssetCatalogRequest(MessageData<AzFramework::AssetSystem::SaveAssetCatalogRequest> messageData)
  350. {
  351. int registrySaveVersion = SaveRegistry();
  352. m_queuedSaveCatalogRequest.insert(registrySaveVersion, messageData.m_key);
  353. }
  354. void AssetCatalog::RegistrySaveComplete(int assetCatalogVersion, bool allCatalogsSaved)
  355. {
  356. for (auto iter = m_queuedSaveCatalogRequest.begin(); iter != m_queuedSaveCatalogRequest.end();)
  357. {
  358. if (iter.key() <= assetCatalogVersion)
  359. {
  360. AssetProcessor::NetworkRequestID& requestId = iter.value();
  361. AzFramework::AssetSystem::SaveAssetCatalogResponse saveCatalogResponse;
  362. saveCatalogResponse.m_saved = allCatalogsSaved;
  363. AssetProcessor::ConnectionBus::Event(requestId.first, &AssetProcessor::ConnectionBus::Events::SendResponse, requestId.second, saveCatalogResponse);
  364. iter = m_queuedSaveCatalogRequest.erase(iter);
  365. }
  366. else
  367. {
  368. ++iter;
  369. }
  370. }
  371. }
  372. int AssetCatalog::SaveRegistry()
  373. {
  374. QMutexLocker locker(&m_savingRegistryMutex);
  375. if (!m_currentlySavingCatalog)
  376. {
  377. m_currentlySavingCatalog = true;
  378. QMetaObject::invokeMethod(this, "SaveRegistry_Impl", Qt::QueuedConnection);
  379. return ++m_currentRegistrySaveVersion;
  380. }
  381. return m_currentRegistrySaveVersion;
  382. }
  383. void AssetCatalog::BuildRegistry()
  384. {
  385. m_catalogIsDirty = true;
  386. m_registryBuiltOnce = true;
  387. {
  388. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  389. QMutexLocker locker(&m_registriesMutex);
  390. for (QString platform : m_platforms)
  391. {
  392. auto inserted = m_registries.insert(platform, AzFramework::AssetRegistry());
  393. AzFramework::AssetRegistry& currentRegistry = inserted.value();
  394. // list of source entries in the database that need to have their UUID updated
  395. AZStd::vector<AzToolsFramework::AssetDatabase::SourceDatabaseEntry> sourceEntriesToUpdate;
  396. QElapsedTimer timer;
  397. timer.start();
  398. auto databaseQueryCallback = [&](AzToolsFramework::AssetDatabase::CombinedDatabaseEntry& combined)
  399. {
  400. SourceAssetReference sourceAsset(combined.m_scanFolderPK, combined.m_scanFolder.c_str(), combined.m_sourceName.c_str());
  401. AZ::Data::AssetId assetId;
  402. auto* fileStateInterface = AZ::Interface<IFileStateRequests>::Get();
  403. if (!fileStateInterface)
  404. {
  405. AZ_Assert(false, "Programmer Error - IFileStateRequests interface is not available");
  406. return false;
  407. }
  408. const bool fileExists = fileStateInterface->Exists(sourceAsset.AbsolutePath().c_str());
  409. // Only try to update for files which actually exist
  410. if (fileExists)
  411. {
  412. auto canonicalUuid = AssetUtilities::GetSourceUuid(sourceAsset);
  413. if (!canonicalUuid)
  414. {
  415. AZ_Error("AssetCatalog", false, "%s", canonicalUuid.GetError().c_str());
  416. return true;
  417. }
  418. assetId = AZ::Data::AssetId(canonicalUuid.GetValue(), combined.m_subID);
  419. if (canonicalUuid.GetValue() != combined.m_sourceGuid)
  420. {
  421. // Canonical UUID does not match stored UUID, this entry needs to be updated
  422. sourceEntriesToUpdate.emplace_back(
  423. combined.m_sourceID,
  424. combined.m_scanFolderID,
  425. combined.m_sourceName.c_str(),
  426. canonicalUuid.GetValue(), // Updated UUID
  427. combined.m_analysisFingerprint.c_str());
  428. }
  429. }
  430. else
  431. {
  432. assetId = AZ::Data::AssetId(combined.m_sourceGuid, combined.m_subID);
  433. }
  434. // relative file path is gotten by removing the platform and game from the product name
  435. AZStd::string_view relativeProductPath = AssetUtilities::StripAssetPlatformNoCopy(combined.m_productName);
  436. QString fullProductPath = m_cacheRoot.absoluteFilePath(combined.m_productName.c_str());
  437. AZ::u64 productFileSize = 0;
  438. AZ::IO::FileIOBase::GetInstance()->Size(fullProductPath.toUtf8().constData(), productFileSize);
  439. AZ::Data::AssetInfo info;
  440. info.m_assetType = combined.m_assetType;
  441. info.m_relativePath = relativeProductPath;
  442. info.m_assetId = assetId;
  443. info.m_sizeBytes = productFileSize;
  444. // also register it at the legacy id(s) if its different:
  445. AZ::Data::AssetId legacyAssetId(combined.m_legacyGuid, 0);
  446. currentRegistry.RegisterAsset(assetId, info);
  447. if (legacyAssetId != assetId)
  448. {
  449. currentRegistry.RegisterLegacyAssetMapping(legacyAssetId, assetId);
  450. }
  451. AZStd::unordered_set<AZ::Data::AssetId> legacySourceAssetIds;
  452. if (fileExists)
  453. {
  454. auto legacySourceUuidsOutcome = AssetUtilities::GetLegacySourceUuids(sourceAsset);
  455. if (legacySourceUuidsOutcome)
  456. {
  457. auto legacySourceUuids = legacySourceUuidsOutcome.GetValue();
  458. legacySourceAssetIds.reserve(legacySourceUuids.size());
  459. for (const auto& legacyUuid : legacySourceUuids)
  460. {
  461. AZ::Data::AssetId legacySourceAssetId(legacyUuid, combined.m_subID);
  462. if (legacySourceAssetId != assetId)
  463. {
  464. legacySourceAssetIds.emplace(legacySourceAssetId);
  465. currentRegistry.RegisterLegacyAssetMapping(legacySourceAssetId, assetId);
  466. }
  467. }
  468. }
  469. }
  470. // now include the additional legacies based on the SubIDs by which this asset was previously referred to.
  471. for (const auto& entry : combined.m_legacySubIDs)
  472. {
  473. AZ::Data::AssetId legacySubID(combined.m_sourceGuid, entry.m_subID);
  474. if ((legacySubID != assetId) && (legacySubID != legacyAssetId) && !legacySourceAssetIds.contains(legacySubID))
  475. {
  476. currentRegistry.RegisterLegacyAssetMapping(legacySubID, assetId);
  477. }
  478. }
  479. return true; // see them all
  480. };
  481. m_db->QueryCombined(
  482. databaseQueryCallback,
  483. AZ::Uuid::CreateNull(),
  484. nullptr,
  485. platform.toUtf8().constData(),
  486. AzToolsFramework::AssetSystem::JobStatus::Any,
  487. true); /*we still need legacy IDs - hardly anyone else does*/
  488. auto* uuidInterface = AZ::Interface<IUuidRequests>::Get();
  489. AZ_Assert(uuidInterface, "Programmer Error - IUuidRequests is not available.");
  490. AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer productDependenciesToUpdate;
  491. m_db->QueryProductDependenciesTable(
  492. [this, &platform, uuidInterface, &productDependenciesToUpdate](AZ::Data::AssetId& assetId, AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry)
  493. {
  494. if (AzFramework::StringFunc::Equal(entry.m_platform.c_str(), platform.toUtf8().data()))
  495. {
  496. // Attempt to update the dependency UUID to the canonical UUID if possible
  497. if (auto canonicalUuid = uuidInterface->GetCanonicalUuid(entry.m_dependencySourceGuid); canonicalUuid && canonicalUuid.value() != entry.m_dependencySourceGuid)
  498. {
  499. entry.m_dependencySourceGuid = canonicalUuid.value();
  500. productDependenciesToUpdate.emplace_back(entry);
  501. }
  502. m_registries[platform].RegisterAssetDependency(
  503. assetId,
  504. AZ::Data::ProductDependency{ AZ::Data::AssetId(entry.m_dependencySourceGuid, entry.m_dependencySubID),
  505. entry.m_dependencyFlags });
  506. }
  507. return true;
  508. });
  509. AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer sourceDependenciesToUpdate;
  510. m_db->QuerySourceDependencies(
  511. [&sourceDependenciesToUpdate, &uuidInterface](AzToolsFramework::AssetDatabase::SourceFileDependencyEntry& entry)
  512. {
  513. bool update = false;
  514. // Check if the sourceGuid needs to be updated
  515. if (auto canonicalUuid = uuidInterface->GetCanonicalUuid(entry.m_sourceGuid);
  516. canonicalUuid && canonicalUuid.value() != entry.m_sourceGuid)
  517. {
  518. if (canonicalUuid.value() != entry.m_sourceGuid)
  519. {
  520. update = true;
  521. entry.m_sourceGuid = canonicalUuid.value();
  522. }
  523. }
  524. // Check if the dependency uses a UUID and if it needs to be updated
  525. if (entry.m_dependsOnSource.IsUuid())
  526. {
  527. if (auto canonicalUuid = uuidInterface->GetCanonicalUuid(entry.m_dependsOnSource.GetUuid());
  528. canonicalUuid && canonicalUuid != entry.m_dependsOnSource.GetUuid())
  529. {
  530. update = true;
  531. entry.m_dependsOnSource = AzToolsFramework::AssetDatabase::PathOrUuid(canonicalUuid.value());
  532. }
  533. }
  534. if (update)
  535. {
  536. sourceDependenciesToUpdate.emplace_back(entry);
  537. }
  538. return true; // Iterate all entries
  539. });
  540. // Update any old source UUIDs
  541. for (auto& sourceDatabaseEntry : sourceEntriesToUpdate)
  542. {
  543. m_db->SetSource(sourceDatabaseEntry);
  544. }
  545. // Update any old product dependencies
  546. for (auto& productDependencyEntry : productDependenciesToUpdate)
  547. {
  548. m_db->SetProductDependency(productDependencyEntry);
  549. }
  550. // Update any old source dependencies
  551. if (!sourceDependenciesToUpdate.empty())
  552. {
  553. m_db->RemoveSourceFileDependencies(sourceDependenciesToUpdate);
  554. m_db->SetSourceFileDependencies(sourceDependenciesToUpdate);
  555. }
  556. AZ_TracePrintf(
  557. "Catalog",
  558. "Read %u assets from database for %s in %fs\n",
  559. currentRegistry.m_assetIdToInfo.size(),
  560. platform.toUtf8().constData(),
  561. timer.elapsed() / 1000.0f);
  562. }
  563. }
  564. Q_EMIT CatalogLoaded();
  565. }
  566. void AssetCatalog::OnDependencyResolved(const AZ::Data::AssetId& assetId, const AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry)
  567. {
  568. QString platform(entry.m_platform.c_str());
  569. if (!m_platforms.contains(platform))
  570. {
  571. return;
  572. }
  573. AzFramework::AssetSystem::AssetNotificationMessage message;
  574. message.m_type = AzFramework::AssetSystem::AssetNotificationMessage::NotificationType::AssetChanged;
  575. // Get the existing data from registry.
  576. AZ::Data::AssetInfo assetInfo = GetAssetInfoById(assetId);
  577. message.m_data = assetInfo.m_relativePath;
  578. message.m_sizeBytes = assetInfo.m_sizeBytes;
  579. message.m_assetId = assetId;
  580. message.m_assetType = assetInfo.m_assetType;
  581. message.m_platform = entry.m_platform.c_str();
  582. // Get legacyIds from registry to put in message.
  583. AZStd::unordered_map<AZ::Data::AssetId, AZ::Data::AssetId> legacyIds;
  584. // Add the new dependency entry and get the list of all dependencies for the message.
  585. AZ::Data::ProductDependency newDependency{ AZ::Data::AssetId(entry.m_dependencySourceGuid, entry.m_dependencySubID), entry.m_dependencyFlags };
  586. {
  587. QMutexLocker locker(&m_registriesMutex);
  588. m_registries[platform].RegisterAssetDependency(assetId, newDependency);
  589. message.m_dependencies = AZStd::move(m_registries[platform].GetAssetDependencies(assetId));
  590. legacyIds = m_registries[platform].GetLegacyMappingSubsetFromRealIds(AZStd::vector<AZ::Data::AssetId>{ assetId });
  591. }
  592. for (auto& legacyId : legacyIds)
  593. {
  594. message.m_legacyAssetIds.emplace_back(legacyId.first);
  595. }
  596. if (m_registryBuiltOnce)
  597. {
  598. Q_EMIT SendAssetMessage(message);
  599. }
  600. m_catalogIsDirty = true;
  601. }
  602. void AssetCatalog::OnConnect(unsigned int connectionId, QStringList platforms)
  603. {
  604. // Send out a message for each asset to make sure the connected tools are aware of the existence of all previously built assets
  605. // since the assetcatalog might not have been written out to disk previously.
  606. for (QString platform : platforms)
  607. {
  608. QMutexLocker locker(&m_registriesMutex);
  609. auto itr = m_registries.find(platform);
  610. if (itr == m_registries.end())
  611. {
  612. continue;
  613. }
  614. const auto& currentRegistry = *itr;
  615. AzFramework::AssetSystem::BulkAssetNotificationMessage bulkMessage;
  616. bulkMessage.m_messages.reserve(currentRegistry.m_assetIdToInfo.size());
  617. bulkMessage.m_type = AzFramework::AssetSystem::AssetNotificationMessage::AssetChanged;
  618. for (const auto& assetInfo : currentRegistry.m_assetIdToInfo)
  619. {
  620. AzFramework::AssetSystem::AssetNotificationMessage message(
  621. assetInfo.second.m_relativePath.c_str(),
  622. AzFramework::AssetSystem::AssetNotificationMessage::AssetChanged,
  623. assetInfo.second.m_assetType,
  624. platform.toUtf8().constData());
  625. message.m_assetId = assetInfo.second.m_assetId;
  626. message.m_sizeBytes = assetInfo.second.m_sizeBytes;
  627. message.m_dependencies = AZStd::move(currentRegistry.GetAssetDependencies(assetInfo.second.m_assetId));
  628. const auto& legacyIds =
  629. m_registries[platform].GetLegacyMappingSubsetFromRealIds(AZStd::vector<AZ::Data::AssetId>{ assetInfo.second.m_assetId });
  630. for (auto& legacyId : legacyIds)
  631. {
  632. message.m_legacyAssetIds.emplace_back(legacyId.first);
  633. }
  634. bulkMessage.m_messages.push_back(AZStd::move(message));
  635. }
  636. AssetProcessor::ConnectionBus::Event(connectionId, &AssetProcessor::ConnectionBus::Events::Send, 0, bulkMessage);
  637. }
  638. }
  639. void AssetCatalog::OnSourceQueued(AZ::Uuid sourceUuid, AZStd::unordered_set<AZ::Uuid> legacyUuids, const SourceAssetReference& sourceAsset)
  640. {
  641. AZStd::lock_guard<AZStd::mutex> lock(m_sourceUUIDToSourceNameMapMutex);
  642. m_sourceUUIDToSourceAssetMap.insert({ sourceUuid, sourceAsset });
  643. //adding legacy source uuid as well
  644. for (const auto legacyUuid : legacyUuids)
  645. {
  646. m_sourceUUIDToSourceAssetMap.insert({ legacyUuid, sourceAsset });
  647. }
  648. m_sourceAssetToSourceUUIDMap.insert({ sourceAsset, sourceUuid });
  649. }
  650. void AssetCatalog::OnSourceFinished(AZ::Uuid sourceUuid, AZStd::unordered_set<AZ::Uuid> legacyUuids)
  651. {
  652. AZStd::lock_guard<AZStd::mutex> lock(m_sourceUUIDToSourceNameMapMutex);
  653. auto found = m_sourceUUIDToSourceAssetMap.find(sourceUuid);
  654. if (found != m_sourceUUIDToSourceAssetMap.end())
  655. {
  656. m_sourceAssetToSourceUUIDMap.erase(found->second);
  657. }
  658. m_sourceUUIDToSourceAssetMap.erase(sourceUuid);
  659. for (const auto& legacyUuid : legacyUuids)
  660. {
  661. m_sourceUUIDToSourceAssetMap.erase(legacyUuid);
  662. }
  663. }
  664. //////////////////////////////////////////////////////////////////////////
  665. bool AssetCatalog::GetRelativeProductPathFromFullSourceOrProductPath(const AZStd::string& fullSourceOrProductPath, AZStd::string& relativeProductPath)
  666. {
  667. ProcessGetRelativeProductPathFromFullSourceOrProductPathRequest(fullSourceOrProductPath, relativeProductPath);
  668. if (!relativeProductPath.length())
  669. {
  670. // if we are here it means we have failed to determine the assetId we will send back the original path
  671. AZ_TracePrintf(AssetProcessor::DebugChannel, "GetRelativeProductPath no result, returning original %s...\n", fullSourceOrProductPath.c_str());
  672. relativeProductPath = fullSourceOrProductPath;
  673. return false;
  674. }
  675. return true;
  676. }
  677. bool AssetCatalog::GenerateRelativeSourcePath(
  678. const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& rootFolder)
  679. {
  680. QString normalizedSourcePath = AssetUtilities::NormalizeFilePath(sourcePath.c_str());
  681. QDir inputPath(normalizedSourcePath);
  682. QString scanFolder;
  683. QString relativeName;
  684. bool validResult = false;
  685. AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessGenerateRelativeSourcePathRequest: %s...\n", sourcePath.c_str());
  686. if (sourcePath.empty())
  687. {
  688. // For an empty input path, do nothing, we'll return an empty, invalid result.
  689. // (We check fullPath instead of inputPath, because an empty fullPath actually produces "." for inputPath)
  690. }
  691. else if (inputPath.isAbsolute())
  692. {
  693. // For an absolute path, try to convert it to a relative path, based on the existing scan folders.
  694. // To get the inputPath, we use absolutePath() instead of path() so that any . or .. entries get collapsed.
  695. validResult = m_platformConfig->ConvertToRelativePath(inputPath.absolutePath(), relativeName, scanFolder);
  696. }
  697. else if (inputPath.isRelative())
  698. {
  699. // For a relative path, concatenate it with each scan folder, and see if a valid relative path emerges.
  700. int scanFolders = m_platformConfig->GetScanFolderCount();
  701. for (int scanIdx = 0; scanIdx < scanFolders; scanIdx++)
  702. {
  703. auto& scanInfo = m_platformConfig->GetScanFolderAt(scanIdx);
  704. QDir possibleRoot(scanInfo.ScanPath());
  705. QDir possibleAbsolutePath = possibleRoot.filePath(normalizedSourcePath);
  706. // To get the inputPath, we use absolutePath() instead of path() so that any . or .. entries get collapsed.
  707. if (m_platformConfig->ConvertToRelativePath(possibleAbsolutePath.absolutePath(), relativeName, scanFolder))
  708. {
  709. validResult = true;
  710. break;
  711. }
  712. }
  713. }
  714. // The input has produced a valid relative path. However, the path might match multiple nested scan folders,
  715. // so look to see if a higher-priority folder has a better match.
  716. if (validResult)
  717. {
  718. QString overridingFile = m_platformConfig->GetOverridingFile(relativeName, scanFolder);
  719. if (!overridingFile.isEmpty())
  720. {
  721. overridingFile = AssetUtilities::NormalizeFilePath(overridingFile);
  722. validResult = m_platformConfig->ConvertToRelativePath(overridingFile, relativeName, scanFolder);
  723. }
  724. }
  725. if (!validResult)
  726. {
  727. // if we are here it means we have failed to determine the relativePath, so we will send back the original path
  728. AZ_TracePrintf(AssetProcessor::DebugChannel,
  729. "GenerateRelativeSourcePath found no valid result, returning original path: %s...\n", sourcePath.c_str());
  730. rootFolder.clear();
  731. relativePath.clear();
  732. relativePath = sourcePath;
  733. return false;
  734. }
  735. relativePath = relativeName.toUtf8().data();
  736. rootFolder = scanFolder.toUtf8().data();
  737. AZ_Assert(!relativePath.empty(), "ConvertToRelativePath returned true, but relativePath is empty");
  738. return true;
  739. }
  740. bool AssetCatalog::GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullSourcePath)
  741. {
  742. ProcessGetFullSourcePathFromRelativeProductPathRequest(relPath, fullSourcePath);
  743. if (!fullSourcePath.length())
  744. {
  745. // if we are here it means that we failed to determine the full source path from the relative path and we will send back the original path
  746. AZ_TracePrintf(AssetProcessor::DebugChannel, "GetFullSourcePath no result, returning original %s...\n", relPath.c_str());
  747. fullSourcePath = relPath;
  748. return false;
  749. }
  750. return true;
  751. }
  752. bool AssetCatalog::GetAssetInfoById(const AZ::Data::AssetId& assetId, const AZ::Data::AssetType& assetType, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, AZStd::string& rootFilePath)
  753. {
  754. assetInfo.m_assetId.SetInvalid();
  755. assetInfo.m_relativePath.clear();
  756. assetInfo.m_assetType = AZ::Data::s_invalidAssetType;
  757. assetInfo.m_sizeBytes = 0;
  758. // If the assetType wasn't provided, try to guess it
  759. if (assetType.IsNull())
  760. {
  761. SourceAssetReference sourceAsset;
  762. bool result = GetAssetInfoByIdOnly(assetId, platformName, assetInfo, sourceAsset);
  763. rootFilePath = sourceAsset.ScanFolderPath().c_str();
  764. return result;
  765. }
  766. bool isSourceType;
  767. {
  768. AZStd::lock_guard<AZStd::mutex> lock(m_sourceAssetTypesMutex);
  769. isSourceType = m_sourceAssetTypes.find(assetType) != m_sourceAssetTypes.end();
  770. }
  771. // If the assetType is registered as a source type, look up the source info
  772. if (isSourceType)
  773. {
  774. SourceAssetReference sourceAsset;
  775. if (GetSourceFileInfoFromAssetId(assetId, sourceAsset))
  776. {
  777. assetInfo.m_assetId = assetId;
  778. assetInfo.m_assetType = assetType;
  779. assetInfo.m_relativePath = sourceAsset.RelativePath().c_str();
  780. AZ::IO::FileIOBase::GetInstance()->Size(sourceAsset.AbsolutePath().c_str(), assetInfo.m_sizeBytes);
  781. rootFilePath = sourceAsset.ScanFolderPath().c_str();
  782. return true;
  783. }
  784. return false;
  785. }
  786. // Return the product file info
  787. rootFilePath.clear(); // products don't have root file paths.
  788. assetInfo = GetProductAssetInfo(platformName.c_str(), assetId);
  789. return !assetInfo.m_relativePath.empty();
  790. }
  791. QString AssetCatalog::GetDefaultAssetPlatform()
  792. {
  793. // get the first available platform, preferring the host platform.
  794. if (m_platforms.contains(AzToolsFramework::AssetSystem::GetHostAssetPlatform()))
  795. {
  796. return QString::fromUtf8(AzToolsFramework::AssetSystem::GetHostAssetPlatform());
  797. }
  798. // the GetHostAssetPlatform() "pc" or "osx" is not actually enabled for this compilation (maybe "server" or similar is in a build job).
  799. // in that case, we'll use the first we find!
  800. return m_platforms[0];
  801. }
  802. AZ::Outcome<AZStd::vector<AZ::Data::ProductDependency>, AZStd::string> AssetCatalog::GetDirectProductDependencies(
  803. const AZ::Data::AssetId& id)
  804. {
  805. QString platform = GetDefaultAssetPlatform();
  806. QMutexLocker locker(&m_registriesMutex);
  807. auto itr = m_registries[platform].m_assetDependencies.find(id);
  808. if (itr == m_registries[platform].m_assetDependencies.end())
  809. {
  810. return AZ::Failure<AZStd::string>("Failed to find asset in dependency map");
  811. }
  812. return AZ::Success(itr->second);
  813. }
  814. AZ::Outcome<AZStd::vector<AZ::Data::ProductDependency>, AZStd::string> AssetCatalog::GetAllProductDependencies(const AZ::Data::AssetId& id)
  815. {
  816. return GetAllProductDependenciesFilter(id, {}, {});
  817. }
  818. AZ::Outcome<AZStd::vector<AZ::Data::ProductDependency>, AZStd::string> AssetCatalog::GetAllProductDependenciesFilter(
  819. const AZ::Data::AssetId& id,
  820. const AZStd::unordered_set<AZ::Data::AssetId>& exclusionList,
  821. const AZStd::vector<AZStd::string>& wildcardPatternExclusionList)
  822. {
  823. AZStd::vector<AZ::Data::ProductDependency> dependencyList;
  824. AZStd::unordered_set<AZ::Data::AssetId> assetSet;
  825. AZ::Data::PreloadAssetListType preloadList;
  826. if (exclusionList.find(id) != exclusionList.end())
  827. {
  828. return AZ::Success(AZStd::move(dependencyList));
  829. }
  830. for (const AZStd::string& wildcardPattern : wildcardPatternExclusionList)
  831. {
  832. if (DoesAssetIdMatchWildcardPattern(id, wildcardPattern))
  833. {
  834. return AZ::Success(AZStd::move(dependencyList));
  835. }
  836. }
  837. AddAssetDependencies(id, assetSet, dependencyList, exclusionList, wildcardPatternExclusionList, preloadList);
  838. // dependencyList will be appended to while looping, so use a traditional loop
  839. for (size_t i = 0; i < dependencyList.size(); ++i)
  840. {
  841. AddAssetDependencies(dependencyList[i].m_assetId, assetSet, dependencyList, exclusionList, wildcardPatternExclusionList, preloadList);
  842. }
  843. return AZ::Success(AZStd::move(dependencyList));
  844. }
  845. AZ::Outcome<AZStd::vector<AZ::Data::ProductDependency>, AZStd::string> AssetCatalog::GetLoadBehaviorProductDependencies(
  846. const AZ::Data::AssetId& id, AZStd::unordered_set<AZ::Data::AssetId>& noloadSet,
  847. AZ::Data::PreloadAssetListType& preloadAssetList)
  848. {
  849. AZStd::vector<AZ::Data::ProductDependency> dependencyList;
  850. AZStd::vector<AZ::Data::ProductDependency> returnList;
  851. AZStd::unordered_set<AZ::Data::AssetId> assetSet;
  852. AddAssetDependencies(id, assetSet, dependencyList, {}, {}, preloadAssetList);
  853. // dependencyList will be appended to while looping, so use a traditional loop
  854. for (size_t i = 0; i < dependencyList.size(); ++i)
  855. {
  856. if (AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(dependencyList[i].m_flags) == AZ::Data::AssetLoadBehavior::NoLoad)
  857. {
  858. noloadSet.insert(dependencyList[i].m_assetId);
  859. assetSet.erase(dependencyList[i].m_assetId);
  860. }
  861. else
  862. {
  863. returnList.push_back(dependencyList[i]);
  864. AddAssetDependencies(dependencyList[i].m_assetId, assetSet, dependencyList, {}, {}, preloadAssetList);
  865. }
  866. }
  867. return AZ::Success(AZStd::move(returnList));
  868. }
  869. bool AssetCatalog::DoesAssetIdMatchWildcardPattern(const AZ::Data::AssetId& assetId, const AZStd::string& wildcardPattern)
  870. {
  871. if (wildcardPattern.empty())
  872. {
  873. // pattern is empty, there is nothing to match
  874. return false;
  875. }
  876. AZStd::string relativePath = GetAssetPathById(assetId);
  877. if (relativePath.empty())
  878. {
  879. // assetId did not resolve to a relative path, cannot be matched
  880. return false;
  881. }
  882. return AZStd::wildcard_match(wildcardPattern, relativePath);
  883. }
  884. void AssetCatalog::AddAssetDependencies(
  885. const AZ::Data::AssetId& searchAssetId,
  886. AZStd::unordered_set<AZ::Data::AssetId>& assetSet,
  887. AZStd::vector<AZ::Data::ProductDependency>& dependencyList,
  888. const AZStd::unordered_set<AZ::Data::AssetId>& exclusionList,
  889. const AZStd::vector<AZStd::string>& wildcardPatternExclusionList,
  890. AZ::Data::PreloadAssetListType& preloadAssetList)
  891. {
  892. using namespace AZ::Data;
  893. QString platform = GetDefaultAssetPlatform();
  894. QMutexLocker locker(&m_registriesMutex);
  895. auto itr = m_registries[platform].m_assetDependencies.find(searchAssetId);
  896. if (itr != m_registries[platform].m_assetDependencies.end())
  897. {
  898. AZStd::vector<ProductDependency>& assetDependencyList = itr->second;
  899. for (const ProductDependency& dependency : assetDependencyList)
  900. {
  901. if (!dependency.m_assetId.IsValid())
  902. {
  903. continue;
  904. }
  905. if (exclusionList.find(dependency.m_assetId) != exclusionList.end())
  906. {
  907. continue;
  908. }
  909. bool isWildcardMatch = false;
  910. for (const AZStd::string& wildcardPattern : wildcardPatternExclusionList)
  911. {
  912. isWildcardMatch = DoesAssetIdMatchWildcardPattern(dependency.m_assetId, wildcardPattern);
  913. if (isWildcardMatch)
  914. {
  915. break;
  916. }
  917. }
  918. if (isWildcardMatch)
  919. {
  920. continue;
  921. }
  922. auto loadBehavior = AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(dependency.m_flags);
  923. if (loadBehavior == AZ::Data::AssetLoadBehavior::PreLoad)
  924. {
  925. preloadAssetList[searchAssetId].insert(dependency.m_assetId);
  926. }
  927. // Only proceed if this ID is valid and we haven't encountered this assetId before.
  928. // Invalid IDs usually come from unmet path product dependencies.
  929. if (assetSet.find(dependency.m_assetId) == assetSet.end())
  930. {
  931. assetSet.insert(dependency.m_assetId); // add to the set of already-encountered assets
  932. dependencyList.push_back(dependency); // put it in the flat list of dependencies we've found
  933. }
  934. }
  935. }
  936. }
  937. bool AssetCatalog::GetSourceInfoBySourcePath(const char* sourcePath, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder)
  938. {
  939. if (!sourcePath || strlen(sourcePath) <= 0)
  940. {
  941. assetInfo.m_assetId.SetInvalid();
  942. return false;
  943. }
  944. SourceAssetReference sourceAsset;
  945. if (!AzFramework::StringFunc::Path::IsRelative(sourcePath))
  946. {
  947. QString scanFolder;
  948. QString relPath;
  949. // Call ConvertToRelativePath first to verify the sourcePath exists in a scanfolder
  950. if (m_platformConfig->ConvertToRelativePath(sourcePath, relPath, scanFolder))
  951. {
  952. sourceAsset = SourceAssetReference(scanFolder, relPath);
  953. }
  954. }
  955. else
  956. {
  957. // relative paths get the first matching asset, and then they get the usual call.
  958. QString absolutePath = m_platformConfig->FindFirstMatchingFile(QString::fromUtf8(sourcePath));
  959. if (!absolutePath.isEmpty())
  960. {
  961. sourceAsset = SourceAssetReference(absolutePath);
  962. }
  963. }
  964. if (!sourceAsset)
  965. {
  966. assetInfo.m_assetId.SetInvalid();
  967. return false;
  968. }
  969. // now that we have a database path, we can at least return something.
  970. // but source info also includes UUID, which we need to hit the database for (or the in-memory map).
  971. // Check the database first for the UUID now that we have the "database name" (which includes output prefix)
  972. {
  973. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  974. AzToolsFramework::AssetDatabase::SourceDatabaseEntry returnedSource;
  975. if (m_db->GetSourceBySourceNameScanFolderId(sourceAsset.RelativePath().c_str(), sourceAsset.ScanFolderId(), returnedSource))
  976. {
  977. const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& entry = returnedSource;
  978. AzToolsFramework::AssetDatabase::ScanFolderDatabaseEntry scanEntry;
  979. if (m_db->GetScanFolderByScanFolderID(entry.m_scanFolderPK, scanEntry))
  980. {
  981. watchFolder = scanEntry.m_scanFolder;
  982. // since we are returning the UUID of a source file, as opposed to the full assetId of a product file produced by that source file,
  983. // the subId part of the assetId will always be set to zero.
  984. assetInfo.m_assetId = AZ::Data::AssetId(entry.m_sourceGuid, 0);
  985. assetInfo.m_relativePath = entry.m_sourceName;
  986. AZStd::string absolutePath;
  987. AzFramework::StringFunc::Path::Join(scanEntry.m_scanFolder.c_str(), assetInfo.m_relativePath.c_str(), absolutePath);
  988. AZ::IO::FileIOBase::GetInstance()->Size(absolutePath.c_str(), assetInfo.m_sizeBytes);
  989. assetInfo.m_assetType = AZ::Uuid::CreateNull(); // most source files don't have a type!
  990. // Go through the list of source assets and see if this asset's file path matches any of the filters
  991. for (const auto& pair : m_sourceAssetTypeFilters)
  992. {
  993. if (AZStd::wildcard_match(pair.first, assetInfo.m_relativePath))
  994. {
  995. assetInfo.m_assetType = pair.second;
  996. break;
  997. }
  998. }
  999. return true;
  1000. }
  1001. }
  1002. }
  1003. watchFolder = sourceAsset.ScanFolderPath().c_str();
  1004. // Source file isn't in the database yet, see if its in the job queue
  1005. if (GetQueuedAssetInfoByRelativeSourceName(sourceAsset, assetInfo))
  1006. {
  1007. return true;
  1008. }
  1009. // Source file isn't in the job queue yet, source UUID needs to be created
  1010. return GetUncachedSourceInfoFromDatabaseNameAndWatchFolder(sourceAsset, assetInfo);
  1011. }
  1012. bool AssetCatalog::GetSourceInfoBySourceUUID(const AZ::Uuid& sourceUuid, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder)
  1013. {
  1014. AZ::Data::AssetId partialId(sourceUuid, 0);
  1015. SourceAssetReference sourceAsset;
  1016. if (GetSourceFileInfoFromAssetId(partialId, sourceAsset))
  1017. {
  1018. watchFolder = sourceAsset.ScanFolderPath().c_str();
  1019. assetInfo.m_assetId = partialId;
  1020. assetInfo.m_assetType = AZ::Uuid::CreateNull(); // most source files don't have a type!
  1021. assetInfo.m_relativePath = sourceAsset.RelativePath().c_str();
  1022. AZ::IO::FileIOBase::GetInstance()->Size(sourceAsset.AbsolutePath().c_str(), assetInfo.m_sizeBytes);
  1023. // if the type has registered with a typeid, then supply it here
  1024. AZStd::lock_guard<AZStd::mutex> lock(m_sourceAssetTypesMutex);
  1025. // Go through the list of source assets and see if this asset's file path matches any of the filters
  1026. // if it does, we know what type it is (if not, the above call to CreateNull ensures it is null).
  1027. for (const auto& pair : m_sourceAssetTypeFilters)
  1028. {
  1029. if (AZStd::wildcard_match(pair.first, sourceAsset.RelativePath().c_str()))
  1030. {
  1031. assetInfo.m_assetType = pair.second;
  1032. break;
  1033. }
  1034. }
  1035. return true;
  1036. }
  1037. // failed!
  1038. return false;
  1039. }
  1040. bool AssetCatalog::GetAssetsProducedBySourceUUID(const AZ::Uuid& sourceUuid, AZStd::vector<AZ::Data::AssetInfo>& productsAssetInfo)
  1041. {
  1042. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1043. AzToolsFramework::AssetDatabase::SourceDatabaseEntry entry;
  1044. if (m_db->GetSourceBySourceGuid(sourceUuid, entry))
  1045. {
  1046. AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer products;
  1047. if (m_db->GetProductsBySourceID(entry.m_sourceID, products))
  1048. {
  1049. for (const AzToolsFramework::AssetDatabase::ProductDatabaseEntry& product : products)
  1050. {
  1051. AZ::Data::AssetInfo assetInfo;
  1052. assetInfo.m_assetId = AZ::Data::AssetId(sourceUuid, product.m_subID);
  1053. assetInfo.m_assetType = product.m_assetType;
  1054. productsAssetInfo.emplace_back(assetInfo);
  1055. }
  1056. }
  1057. return true;
  1058. }
  1059. return false;
  1060. }
  1061. bool AssetCatalog::ClearFingerprintForAsset(const AZStd::string& sourcePath)
  1062. {
  1063. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1064. SourceAssetReference sourceAsset;
  1065. if (QFileInfo(sourcePath.c_str()).isAbsolute())
  1066. {
  1067. sourceAsset = SourceAssetReference(sourcePath.c_str());
  1068. }
  1069. else
  1070. {
  1071. QString absolutePath = m_platformConfig->FindFirstMatchingFile(sourcePath.c_str());
  1072. if (absolutePath.isEmpty())
  1073. {
  1074. return false;
  1075. }
  1076. sourceAsset = SourceAssetReference(absolutePath.toUtf8().constData());
  1077. }
  1078. if(!m_db->UpdateFileHashByFileNameAndScanFolderId(sourceAsset.RelativePath().c_str(), sourceAsset.ScanFolderId(), 0))
  1079. {
  1080. return false;
  1081. }
  1082. AzToolsFramework::AssetDatabase::SourceDatabaseEntry source;
  1083. if (!m_db->GetSourceBySourceNameScanFolderId(sourceAsset.RelativePath().c_str(), sourceAsset.ScanFolderId(), source))
  1084. {
  1085. return false;
  1086. }
  1087. // if setting the file hash failed, still try to clear the job fingerprints.
  1088. return m_db->SetJobFingerprintsBySourceID(source.m_sourceID, 0);
  1089. }
  1090. bool AssetCatalog::GetScanFolders(AZStd::vector<AZStd::string>& scanFolders)
  1091. {
  1092. int scanFolderCount = m_platformConfig->GetScanFolderCount();
  1093. for (int i = 0; i < scanFolderCount; ++i)
  1094. {
  1095. scanFolders.push_back(m_platformConfig->GetScanFolderAt(i).ScanPath().toUtf8().constData());
  1096. }
  1097. return true;
  1098. }
  1099. bool AssetCatalog::GetAssetSafeFolders(AZStd::vector<AZStd::string>& assetSafeFolders)
  1100. {
  1101. int scanFolderCount = m_platformConfig->GetScanFolderCount();
  1102. for (int scanFolderIndex = 0; scanFolderIndex < scanFolderCount; ++scanFolderIndex)
  1103. {
  1104. AssetProcessor::ScanFolderInfo& scanFolder = m_platformConfig->GetScanFolderAt(scanFolderIndex);
  1105. if (scanFolder.CanSaveNewAssets())
  1106. {
  1107. assetSafeFolders.push_back(scanFolder.ScanPath().toUtf8().constData());
  1108. }
  1109. }
  1110. return true;
  1111. }
  1112. bool AssetCatalog::IsAssetPlatformEnabled(const char* platform)
  1113. {
  1114. const AZStd::vector<AssetBuilderSDK::PlatformInfo>& enabledPlatforms = m_platformConfig->GetEnabledPlatforms();
  1115. for (const AssetBuilderSDK::PlatformInfo& platformInfo : enabledPlatforms)
  1116. {
  1117. if (platformInfo.m_identifier == platform)
  1118. {
  1119. return true;
  1120. }
  1121. }
  1122. return false;
  1123. }
  1124. int AssetCatalog::GetPendingAssetsForPlatform(const char* /*platform*/)
  1125. {
  1126. AZ_Assert(false, "Call to unsupported Asset Processor function GetPendingAssetsForPlatform on AssetCatalog");
  1127. return -1;
  1128. }
  1129. AZStd::string AssetCatalog::GetAssetPathById(const AZ::Data::AssetId& id)
  1130. {
  1131. return GetAssetInfoById(id).m_relativePath;
  1132. }
  1133. AZ::Data::AssetId AssetCatalog::GetAssetIdByPath(const char* path, const AZ::Data::AssetType& typeToRegister, bool autoRegisterIfNotFound)
  1134. {
  1135. AZ_UNUSED(autoRegisterIfNotFound);
  1136. AZ_Assert(autoRegisterIfNotFound == false, "Auto registration is invalid during asset processing.");
  1137. AZ_UNUSED(typeToRegister);
  1138. AZ_Assert(typeToRegister == AZ::Data::s_invalidAssetType, "Can not register types during asset processing.");
  1139. AZStd::string relProductPath;
  1140. GetRelativeProductPathFromFullSourceOrProductPath(path, relProductPath);
  1141. QString tempPlatformName = GetDefaultAssetPlatform();
  1142. AZ::Data::AssetId assetId;
  1143. {
  1144. QMutexLocker locker(&m_registriesMutex);
  1145. assetId = m_registries[tempPlatformName].GetAssetIdByPath(relProductPath.c_str());
  1146. }
  1147. return assetId;
  1148. }
  1149. AZ::Data::AssetInfo AssetCatalog::GetAssetInfoById(const AZ::Data::AssetId& id)
  1150. {
  1151. AZ::Data::AssetType assetType;
  1152. AZ::Data::AssetInfo assetInfo;
  1153. AZStd::string rootFilePath;
  1154. GetAssetInfoById(id, assetType, "", assetInfo, rootFilePath);
  1155. return assetInfo;
  1156. }
  1157. bool ConvertDatabaseProductPathToProductFilename(AZStd::string_view dbPath, QString& productFileName)
  1158. {
  1159. // Always strip the leading directory from the product path
  1160. // The leading directory can be either an asset platform path or a subfolder
  1161. AZ::StringFunc::TokenizeNext(dbPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR);
  1162. if (!dbPath.empty())
  1163. {
  1164. productFileName = QString::fromUtf8(dbPath.data(), aznumeric_cast<int>(dbPath.size()));
  1165. return true;
  1166. }
  1167. return false;
  1168. }
  1169. void AssetCatalog::ProcessGetRelativeProductPathFromFullSourceOrProductPathRequest(const AZStd::string& fullPath, AZStd::string& relativeProductPath)
  1170. {
  1171. QString sourceOrProductPath = fullPath.c_str();
  1172. QString normalizedSourceOrProductPath = AssetUtilities::NormalizeFilePath(sourceOrProductPath);
  1173. QString productFileName;
  1174. bool resultCode = false;
  1175. QDir inputPath(normalizedSourceOrProductPath);
  1176. AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessGetRelativeProductPath: %s...\n", sourceOrProductPath.toUtf8().constData());
  1177. if (inputPath.isRelative())
  1178. {
  1179. //if the path coming in is already a relative path,we just send it back
  1180. productFileName = sourceOrProductPath;
  1181. resultCode = true;
  1182. }
  1183. else
  1184. {
  1185. QDir cacheRoot;
  1186. AssetUtilities::ComputeProjectCacheRoot(cacheRoot);
  1187. QString normalizedCacheRoot = AssetUtilities::NormalizeFilePath(cacheRoot.path());
  1188. if (AssetUtilities::IsInCacheFolder(normalizedSourceOrProductPath.toUtf8().constData(), cacheRoot.absolutePath().toUtf8().constData()))
  1189. {
  1190. // The path send by the game/editor contains the cache root so we try to find the asset id
  1191. // from the asset database
  1192. normalizedSourceOrProductPath.remove(0, normalizedCacheRoot.length() + 1); // adding 1 for the native separator
  1193. // If we are here it means that the asset database does not have any knowledge about this file,
  1194. // most probably because AP has not processed the file yet
  1195. // In this case we will try to compute the asset id from the product path
  1196. // Now after removing the cache root,normalizedInputAssetPath can either be $Platform/$Game/xxx/yyy or something like $Platform/zzz
  1197. // and the corresponding assetId have to be either xxx/yyy or zzz
  1198. resultCode = ConvertDatabaseProductPathToProductFilename(normalizedSourceOrProductPath.toUtf8().data(), productFileName);
  1199. }
  1200. else
  1201. {
  1202. // If we are here it means its a source file, first see whether there is any overriding file and then try to find products
  1203. QString scanFolder;
  1204. QString relativeName;
  1205. if (m_platformConfig->ConvertToRelativePath(normalizedSourceOrProductPath, relativeName, scanFolder))
  1206. {
  1207. QString overridingFile = m_platformConfig->GetOverridingFile(relativeName, scanFolder);
  1208. if (overridingFile.isEmpty())
  1209. {
  1210. // no overriding file found
  1211. overridingFile = normalizedSourceOrProductPath;
  1212. }
  1213. else
  1214. {
  1215. overridingFile = AssetUtilities::NormalizeFilePath(overridingFile);
  1216. }
  1217. const auto* scanFolderInfo = m_platformConfig->GetScanFolderForFile(overridingFile);
  1218. if (scanFolderInfo && m_platformConfig->ConvertToRelativePath(overridingFile, scanFolderInfo, relativeName))
  1219. {
  1220. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1221. AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer products;
  1222. if (m_db->GetProductsBySourceNameScanFolderID(relativeName, scanFolderInfo->ScanFolderID(), products))
  1223. {
  1224. resultCode = ConvertDatabaseProductPathToProductFilename(products[0].m_productName, productFileName);
  1225. }
  1226. else
  1227. {
  1228. productFileName = relativeName;
  1229. resultCode = true;
  1230. }
  1231. }
  1232. }
  1233. }
  1234. }
  1235. if (!resultCode)
  1236. {
  1237. productFileName = "";
  1238. }
  1239. relativeProductPath = productFileName.toUtf8().data();
  1240. }
  1241. void AssetCatalog::ProcessGetFullSourcePathFromRelativeProductPathRequest(const AZStd::string& relPath, AZStd::string& fullSourcePath)
  1242. {
  1243. QString assetPath = relPath.c_str();
  1244. QString normalizedAssetPath = AssetUtilities::NormalizeFilePath(assetPath);
  1245. int resultCode = 0;
  1246. QString fullAssetPath;
  1247. if (normalizedAssetPath.isEmpty())
  1248. {
  1249. fullSourcePath = "";
  1250. return;
  1251. }
  1252. QDir inputPath(normalizedAssetPath);
  1253. if (inputPath.isAbsolute())
  1254. {
  1255. QDir cacheRoot;
  1256. AssetUtilities::ComputeProjectCacheRoot(cacheRoot);
  1257. QString normalizedCacheRoot = AssetUtilities::NormalizeFilePath(cacheRoot.path());
  1258. if (!AssetUtilities::IsInCacheFolder(normalizedAssetPath.toUtf8().constData(), cacheRoot.absolutePath().toUtf8().constData()))
  1259. {
  1260. // Attempt to convert to relative path
  1261. QString dummy, convertedRelPath;
  1262. if (m_platformConfig->ConvertToRelativePath(assetPath, convertedRelPath, dummy))
  1263. {
  1264. // then find the first matching file to get correct casing
  1265. fullAssetPath = m_platformConfig->FindFirstMatchingFile(convertedRelPath);
  1266. }
  1267. if (fullAssetPath.isEmpty())
  1268. {
  1269. // if we couldn't find it, just return the passed in path
  1270. fullAssetPath = assetPath;
  1271. }
  1272. resultCode = 1;
  1273. }
  1274. else
  1275. {
  1276. // The path send by the game/editor contains the cache root ,try to find the productName from it
  1277. normalizedAssetPath.remove(0, normalizedCacheRoot.length() + 1); // adding 1 for the native separator
  1278. }
  1279. }
  1280. if (!resultCode)
  1281. {
  1282. //remove aliases if present
  1283. normalizedAssetPath = AssetUtilities::NormalizeAndRemoveAlias(normalizedAssetPath);
  1284. if (!normalizedAssetPath.isEmpty()) // this happens if it comes in as just for example "@products@/"
  1285. {
  1286. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1287. //We should have the asset now, we can now find the full asset path
  1288. // we have to check each platform individually until we get a hit.
  1289. const auto& platforms = m_platformConfig->GetEnabledPlatforms();
  1290. QString productName;
  1291. for (const AssetBuilderSDK::PlatformInfo& platformInfo : platforms)
  1292. {
  1293. if (platformInfo.m_identifier == AssetBuilderSDK::CommonPlatformName)
  1294. {
  1295. // Common platform is not supported for product assets currently
  1296. continue;
  1297. }
  1298. QString platformName = QString::fromUtf8(platformInfo.m_identifier.c_str());
  1299. productName = AssetUtilities::GuessProductNameInDatabase(normalizedAssetPath, platformName, m_db.get());
  1300. if (!productName.isEmpty())
  1301. {
  1302. break;
  1303. }
  1304. }
  1305. if (!productName.isEmpty())
  1306. {
  1307. //Now find the input name for the path,if we are here this should always return true since we were able to find the productName before
  1308. AzToolsFramework::AssetDatabase::SourceDatabaseEntryContainer sources;
  1309. if (m_db->GetSourcesByProductName(productName, sources))
  1310. {
  1311. //Once we have found the inputname we will try finding the full path
  1312. fullAssetPath = m_platformConfig->FindFirstMatchingFile(sources[0].m_sourceName.c_str());
  1313. if (!fullAssetPath.isEmpty())
  1314. {
  1315. resultCode = 1;
  1316. }
  1317. }
  1318. }
  1319. else
  1320. {
  1321. // if we are not able to guess the product name than maybe the asset path is an input name
  1322. fullAssetPath = m_platformConfig->FindFirstMatchingFile(normalizedAssetPath);
  1323. if (!fullAssetPath.isEmpty())
  1324. {
  1325. resultCode = 1;
  1326. }
  1327. }
  1328. }
  1329. }
  1330. if (!resultCode)
  1331. {
  1332. fullSourcePath = "";
  1333. }
  1334. else
  1335. {
  1336. fullSourcePath = fullAssetPath.toUtf8().data();
  1337. }
  1338. }
  1339. //////////////////////////////////////////////////////////////////////////
  1340. void AssetCatalog::RegisterSourceAssetType(const AZ::Data::AssetType& assetType, const char* assetFileFilter)
  1341. {
  1342. AZStd::lock_guard<AZStd::mutex> lock(m_sourceAssetTypesMutex);
  1343. m_sourceAssetTypes.insert(assetType);
  1344. AZStd::vector<AZStd::string> tokens;
  1345. AZStd::string semicolonSeperated(assetFileFilter);
  1346. AZStd::tokenize(semicolonSeperated, AZStd::string(";"), tokens);
  1347. for (const auto& pattern : tokens)
  1348. {
  1349. m_sourceAssetTypeFilters[pattern] = assetType;
  1350. }
  1351. }
  1352. void AssetCatalog::UnregisterSourceAssetType(const AZ::Data::AssetType& /*assetType*/)
  1353. {
  1354. // For now, this does nothing, because it would just needlessly complicate things for no gain.
  1355. // Unregister is only called when a builder is shut down, which really is only supposed to happen when AssetCatalog is being shutdown
  1356. // Without a way of tracking how many builders have registered the same assetType and being able to perfectly keep track of every builder shutdown, even in the event of a crash,
  1357. // the map would either be cleared prematurely or never get cleared at all
  1358. }
  1359. //////////////////////////////////////////////////////////////////////////
  1360. bool AssetCatalog::GetSourceFileInfoFromAssetId(const AZ::Data::AssetId &assetId, SourceAssetReference& sourceAsset)
  1361. {
  1362. // Try checking the UuidManager, it keeps track of legacy UUIDs
  1363. auto* uuidInterface = AZ::Interface<AssetProcessor::IUuidRequests>::Get();
  1364. AZ_Assert(uuidInterface, "Programmer Error - IUuidRequests interface is not available.");
  1365. if (auto result = uuidInterface->FindHighestPriorityFileByUuid(assetId.m_guid); result)
  1366. {
  1367. sourceAsset = SourceAssetReference(result.value());
  1368. return true;
  1369. }
  1370. // Check the database next
  1371. {
  1372. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1373. AzToolsFramework::AssetDatabase::SourceDatabaseEntry entry;
  1374. if (m_db->GetSourceBySourceGuid(assetId.m_guid, entry))
  1375. {
  1376. AzToolsFramework::AssetDatabase::ScanFolderDatabaseEntry scanEntry;
  1377. if (m_db->GetScanFolderByScanFolderID(entry.m_scanFolderPK, scanEntry))
  1378. {
  1379. sourceAsset = SourceAssetReference(scanEntry.m_scanFolder.c_str(), entry.m_sourceName.c_str());
  1380. return true;
  1381. }
  1382. }
  1383. }
  1384. // Source file isn't in the database yet, see if its in the job queue
  1385. return GetQueuedAssetInfoById(assetId.m_guid, sourceAsset);
  1386. }
  1387. AZ::Data::AssetInfo AssetCatalog::GetProductAssetInfo(const char* platformName, const AZ::Data::AssetId& assetId)
  1388. {
  1389. // this more or less follows the same algorithm that the game uses to look up products.
  1390. using namespace AZ::Data;
  1391. using namespace AzFramework;
  1392. if ((!assetId.IsValid()) || (m_platforms.isEmpty()))
  1393. {
  1394. return AssetInfo();
  1395. }
  1396. // in case no platform name has been given, we are prepared to compute one.
  1397. QString tempPlatformName;
  1398. // if no platform specified, we'll use the current platform.
  1399. if ((!platformName) || (platformName[0] == 0))
  1400. {
  1401. tempPlatformName = GetDefaultAssetPlatform();
  1402. }
  1403. else
  1404. {
  1405. tempPlatformName = QString::fromUtf8(platformName);
  1406. }
  1407. // note that m_platforms is not mutated at all during runtime, so we ignore it in the lock
  1408. if (!m_platforms.contains(tempPlatformName))
  1409. {
  1410. return AssetInfo();
  1411. }
  1412. QMutexLocker locker(&m_registriesMutex);
  1413. const AssetRegistry& registryToUse = m_registries[tempPlatformName];
  1414. auto foundIter = registryToUse.m_assetIdToInfo.find(assetId);
  1415. if (foundIter != registryToUse.m_assetIdToInfo.end())
  1416. {
  1417. return foundIter->second;
  1418. }
  1419. // we did not find it - try the backup mapping!
  1420. AssetId legacyMapping = registryToUse.GetAssetIdByLegacyAssetId(assetId);
  1421. if (legacyMapping.IsValid())
  1422. {
  1423. return GetProductAssetInfo(platformName, legacyMapping);
  1424. }
  1425. return AssetInfo(); // not found!
  1426. }
  1427. bool AssetCatalog::GetAssetInfoByIdOnly(const AZ::Data::AssetId& id, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, SourceAssetReference& sourceAsset)
  1428. {
  1429. if (GetSourceFileInfoFromAssetId(id, sourceAsset))
  1430. {
  1431. {
  1432. AZStd::lock_guard<AZStd::mutex> lock(m_sourceAssetTypesMutex);
  1433. // Go through the list of source assets and see if this asset's file path matches any of the filters
  1434. for (const auto& pair : m_sourceAssetTypeFilters)
  1435. {
  1436. if (AZStd::wildcard_match(pair.first, sourceAsset.AbsolutePath().c_str()))
  1437. {
  1438. assetInfo.m_assetId = id;
  1439. assetInfo.m_assetType = pair.second;
  1440. assetInfo.m_relativePath = sourceAsset.RelativePath().c_str();
  1441. AZ::IO::FileIOBase::GetInstance()->Size(sourceAsset.AbsolutePath().c_str(), assetInfo.m_sizeBytes);
  1442. return true;
  1443. }
  1444. }
  1445. }
  1446. // If we get to here, we're going to assume it's a product type
  1447. sourceAsset = {};
  1448. assetInfo = GetProductAssetInfo(platformName.c_str(), id);
  1449. return !assetInfo.m_relativePath.empty();
  1450. }
  1451. // Asset isn't in the DB or in the APM queue, we don't know what this asset ID is
  1452. return false;
  1453. }
  1454. bool AssetCatalog::GetQueuedAssetInfoById(const AZ::Uuid& guid, SourceAssetReference& sourceAsset)
  1455. {
  1456. if (!guid.IsNull())
  1457. {
  1458. AZStd::lock_guard<AZStd::mutex> lock(m_sourceUUIDToSourceNameMapMutex);
  1459. auto foundSource = m_sourceUUIDToSourceAssetMap.find(guid);
  1460. if (foundSource != m_sourceUUIDToSourceAssetMap.end())
  1461. {
  1462. sourceAsset = foundSource->second;
  1463. return true;
  1464. }
  1465. AZ_TracePrintf(AssetProcessor::DebugChannel, "GetQueuedAssetInfoById: AssetCatalog unable to find the requested source asset having uuid (%s).\n", guid.ToString<AZStd::string>().c_str());
  1466. }
  1467. return false;
  1468. }
  1469. bool AssetCatalog::GetQueuedAssetInfoByRelativeSourceName(const SourceAssetReference& sourceAsset, AZ::Data::AssetInfo& assetInfo)
  1470. {
  1471. if (sourceAsset)
  1472. {
  1473. AZStd::lock_guard<AZStd::mutex> lock(m_sourceUUIDToSourceNameMapMutex);
  1474. auto foundSourceUUID = m_sourceAssetToSourceUUIDMap.find(sourceAsset);
  1475. if (foundSourceUUID != m_sourceAssetToSourceUUIDMap.end())
  1476. {
  1477. assetInfo.m_relativePath = sourceAsset.RelativePath().c_str();
  1478. assetInfo.m_assetId = foundSourceUUID->second;
  1479. AZ::IO::FileIOBase::GetInstance()->Size(sourceAsset.AbsolutePath().c_str(), assetInfo.m_sizeBytes);
  1480. assetInfo.m_assetType = AZ::Uuid::CreateNull(); // most source files don't have a type!
  1481. // Go through the list of source assets and see if this asset's file path matches any of the filters
  1482. for (const auto& pair : m_sourceAssetTypeFilters)
  1483. {
  1484. if (AZStd::wildcard_match(pair.first, assetInfo.m_relativePath))
  1485. {
  1486. assetInfo.m_assetType = pair.second;
  1487. break;
  1488. }
  1489. }
  1490. return true;
  1491. }
  1492. }
  1493. assetInfo.m_assetId.SetInvalid();
  1494. return false;
  1495. }
  1496. bool AssetCatalog::GetUncachedSourceInfoFromDatabaseNameAndWatchFolder(const SourceAssetReference& sourceAsset, AZ::Data::AssetInfo& assetInfo)
  1497. {
  1498. // Make sure the source file exists first
  1499. AZ::IO::FileIOBase* io = AZ::IO::FileIOBase::GetInstance();
  1500. AZ_Assert(io, "Expected a FileIO Interface");
  1501. if (!io->Exists(sourceAsset.AbsolutePath().c_str()))
  1502. {
  1503. return false;
  1504. }
  1505. auto sourceUUID = AssetUtilities::GetSourceUuid(sourceAsset);
  1506. if (!sourceUUID)
  1507. {
  1508. return false;
  1509. }
  1510. AZ::Data::AssetId sourceAssetId(sourceUUID.GetValue(), 0);
  1511. assetInfo.m_assetId = sourceAssetId;
  1512. assetInfo.m_relativePath = sourceAsset.RelativePath().c_str();
  1513. io->Size(sourceAsset.AbsolutePath().c_str(), assetInfo.m_sizeBytes);
  1514. assetInfo.m_assetType = AZ::Uuid::CreateNull();
  1515. // Go through the list of source assets and see if this asset's file path matches any of the filters
  1516. for (const auto& pair : m_sourceAssetTypeFilters)
  1517. {
  1518. if (AZStd::wildcard_match(pair.first, assetInfo.m_relativePath))
  1519. {
  1520. assetInfo.m_assetType = pair.second;
  1521. break;
  1522. }
  1523. }
  1524. return true;
  1525. }
  1526. bool AssetCatalog::ConnectToDatabase()
  1527. {
  1528. AZStd::lock_guard<AZStd::mutex> lock(m_databaseMutex);
  1529. if (!m_db)
  1530. {
  1531. AZStd::string databaseLocation;
  1532. AzToolsFramework::AssetDatabase::AssetDatabaseRequestsBus::Broadcast(&AzToolsFramework::AssetDatabase::AssetDatabaseRequests::GetAssetDatabaseLocation, databaseLocation);
  1533. if (!databaseLocation.empty())
  1534. {
  1535. m_db = AZStd::make_unique<AssetProcessor::AssetDatabaseConnection>();
  1536. m_db->OpenDatabase();
  1537. return true;
  1538. }
  1539. return false;
  1540. }
  1541. return true;
  1542. }
  1543. void AssetCatalog::AsyncAssetCatalogStatusRequest()
  1544. {
  1545. if (m_catalogIsDirty)
  1546. {
  1547. Q_EMIT AsyncAssetCatalogStatusResponse(AssetCatalogStatus::RequiresSaving);
  1548. }
  1549. else
  1550. {
  1551. Q_EMIT AsyncAssetCatalogStatusResponse(AssetCatalogStatus::UpToDate);
  1552. }
  1553. }
  1554. }//namespace AssetProcessor