Optimize initial AP startup time

Added a string path cache to the FileStateCache to avoid re-normalizing and lowercasing the same string many times.  Profiling showed this as a hot spot and it was being called over 5M times during startup.
Added a dependency cache to QueryAbsolutePathDependenciesRecursive.  Profiling also showed this as a hotspot for several different items.  The cache only applies during startup and is cleared/disabled after reaching idle state.
Added an optional return of the scanfolder for FindFirstMatchingFile to optimize one of the SourceAssetReference calls during the above.

Signed-off-by: amzn-mike <[email protected]>

Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp

@@ -42,7 +42,7 @@ namespace AssetProcessor
         LockGuardType scopeLock(m_mapMutex);
         QString key = PathToKey(existingInfo.m_filePath);
         m_fileInfoMap[key] = FileStateInfo(existingInfo);
-        
+
         // it is possible to update the cache so that the info is known, but the hash is not.
         if (hash == InvalidFileHash)
         {
@@ -151,6 +151,8 @@ namespace AssetProcessor
 
     void FileStateCache::InvalidateHash(const QString& absolutePath)
     {
+        m_keyCache = {}; // Clear the key cache, its only really intended to help speedup the startup phase
+
         auto fileHashItr = m_fileHashMap.find(PathToKey(absolutePath));
 
         if (fileHashItr != m_fileHashMap.end())
@@ -163,16 +165,22 @@ namespace AssetProcessor
 
     QString FileStateCache::PathToKey(const QString& absolutePath) const
     {
-        QString normalized = AssetUtilities::NormalizeFilePath(absolutePath);
+        auto cached = m_keyCache.find(absolutePath);
 
-        if constexpr (!ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM)
+        if (cached != m_keyCache.end())
         {
-            return normalized.toLower();
+            return cached.value();
         }
-        else
+
+        QString normalized = AssetUtilities::NormalizeFilePath(absolutePath);
+
+        if constexpr (!ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM)
         {
-            return normalized;
+            normalized = normalized.toLower();
         }
+
+        m_keyCache[absolutePath] = normalized;
+        return normalized;
     }
 
     void FileStateCache::AddOrUpdateFileInternal(QFileInfo fileInfo)

Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.h

@@ -15,6 +15,7 @@
 #include <QFileInfo>
 #include <AzCore/Interface/Interface.h>
 #include <AzCore/EBus/Event.h>
+#include <AzCore/IO/FileIO.h>
 
 namespace AssetProcessor
 {
@@ -55,7 +56,7 @@ namespace AssetProcessor
         /// Convenience function to check if a file or directory exists.
         virtual bool Exists(const QString& absolutePath) const = 0;
         virtual bool GetHash(const QString& absolutePath, FileHash* foundHash) = 0;
-        
+
         //! Called when the caller knows a hash and file info already.
         //! This can for example warm up the cache so that it can return hashes without actually hashing.
         //! (optional for implementations)
@@ -90,7 +91,7 @@ namespace AssetProcessor
 
         /// Removes a file from the cache
         virtual void RemoveFile(const QString& /*absolutePath*/) {}
-        
+
         virtual void WarmUpCache(const AssetFileInfo& /*existingInfo*/, const FileHash /*hash*/) {}
     };
 
@@ -111,7 +112,7 @@ namespace AssetProcessor
         void AddFile(const QString& absolutePath) override;
         void UpdateFile(const QString& absolutePath) override;
         void RemoveFile(const QString& absolutePath) override;
-        
+
         void WarmUpCache(const AssetFileInfo& existingInfo, const FileHash hash = IFileStateRequests::InvalidFileHash) override;
 
     private:
@@ -135,6 +136,10 @@ namespace AssetProcessor
 
         AZ::Event<FileStateInfo> m_deleteEvent;
 
+        /// Cache of input path values to their final, normalized map key format.
+        /// Profiling has shown path normalization to be a hotspot.
+        mutable QHash<QString, QString> m_keyCache;
+
         using LockGuardType = AZStd::lock_guard<decltype(m_mapMutex)>;
     };
 

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp

@@ -37,6 +37,7 @@
 #include <AzToolsFramework/Metadata/MetadataManager.h>
 #include <native/utilities/UuidManager.h>
 #include <native/utilities/ProductOutputUtil.h>
+#include <native/AssetManager/FileStateCache.h>
 
 namespace AssetProcessor
 {
@@ -3210,6 +3211,8 @@ namespace AssetProcessor
                 AZ_TracePrintf(ConsoleChannel, "Builder optimization: %i / %i files required full analysis, %i sources found but not processed by anyone\n", m_numSourcesNeedingFullAnalysis, m_numTotalSourcesFound, m_numSourcesNotHandledByAnyBuilder);
             }
 
+            m_dependencyCache = {};
+            m_dependencyCacheEnabled = false;
             m_pathDependencyManager->ProcessQueuedDependencyResolves();
             QTimer::singleShot(20, this, SLOT(RemoveEmptyFolders()));
         }
@@ -5584,13 +5587,13 @@ namespace AssetProcessor
 
         // then we add database dependencies.  We have to query this recursively so that we get dependencies of dependencies:
         AZStd::unordered_set<PathOrUuid> results;
-        AZStd::queue<PathOrUuid> queryQueue;
-        queryQueue.push(PathOrUuid(sourceUuid));
+        AZStd::unordered_set<PathOrUuid> queryQueue;
+        queryQueue.emplace(PathOrUuid(sourceUuid));
 
         while (!queryQueue.empty())
         {
-            PathOrUuid toSearch = queryQueue.front();
-            queryQueue.pop();
+            PathOrUuid toSearch = *queryQueue.begin();
+            queryQueue.erase(toSearch);
 
             // if we've already queried it, dont do it again (breaks recursion)
             if (results.contains(toSearch))
@@ -5609,20 +5612,23 @@ namespace AssetProcessor
                 // If the dependency is not an asset, this will resolve to an invalid UUID which will simply return no results for our
                 // search
 
-                QString absolutePath;
-
                 if (AZ::IO::PathView(toSearch.GetPath()).IsAbsolute())
                 {
-                    absolutePath = toSearch.GetPath().c_str();
+                    if (AZ::Interface<IFileStateRequests>::Get()->Exists(toSearch.GetPath().c_str()))
+                    {
+                        searchUuid = AssetUtilities::GetSourceUuid(SourceAssetReference(toSearch.GetPath().c_str())).GetValueOr(AZ::Uuid());
+                    }
                 }
                 else
                 {
-                    absolutePath = m_platformConfig->FindFirstMatchingFile(toSearch.GetPath().c_str());
-                }
+                    const ScanFolderInfo* scanFolder = nullptr;
+                    m_platformConfig->FindFirstMatchingFile(toSearch.GetPath().c_str(), false, &scanFolder);
 
-                if (AZ::IO::FileIOBase::GetInstance()->Exists(absolutePath.toUtf8().constData()))
-                {
-                    searchUuid = AssetUtilities::GetSourceUuid(SourceAssetReference(absolutePath.toUtf8().constData())).GetValueOr(AZ::Uuid());
+                    if (scanFolder)
+                    {
+                        searchUuid = AssetUtilities::GetSourceUuid(SourceAssetReference(
+                            scanFolder->ScanFolderID(), scanFolder->ScanPath().toUtf8().constData(), toSearch.GetPath().c_str())).GetValueOr(AZ::Uuid());
+                    }
                 }
             }
             else
@@ -5630,13 +5636,30 @@ namespace AssetProcessor
                 searchUuid = toSearch.GetUuid();
             }
 
-            auto callbackFunction = [&queryQueue](SourceFileDependencyEntry& entry)
+            auto cacheEntry = m_dependencyCache.find(searchUuid);
+
+            if (dependencyType == AzToolsFramework::AssetDatabase::SourceFileDependencyEntry::TypeOfDependency::DEP_Any &&
+                cacheEntry != m_dependencyCache.end())
             {
-                queryQueue.push(entry.m_dependsOnSource);
-                return true;
-            };
+                queryQueue.insert(cacheEntry->second.begin(), cacheEntry->second.end());
+            }
+            else
+            {
+                auto callbackFunction = [this, &queryQueue, dependencyType](SourceFileDependencyEntry& entry)
+                {
+                    queryQueue.emplace(entry.m_dependsOnSource);
+
+                    if (m_dependencyCacheEnabled &&
+                        dependencyType == AzToolsFramework::AssetDatabase::SourceFileDependencyEntry::TypeOfDependency::DEP_Any)
+                    {
+                        m_dependencyCache[entry.m_sourceGuid].emplace_back(entry.m_dependsOnSource);
+                    }
+
+                    return true;
+                };
 
-            m_stateData->QueryDependsOnSourceBySourceDependency(searchUuid, dependencyType, callbackFunction);
+                m_stateData->QueryDependsOnSourceBySourceDependency(searchUuid, dependencyType, callbackFunction);
+            }
         }
 
         for (const PathOrUuid& dep : results)
@@ -5674,6 +5697,7 @@ namespace AssetProcessor
             finalDependencyList.insert(AZStd::make_pair(absolutePath.toUtf8().constData(), dep.ToString().c_str()));
         }
     }
+
     bool AssetProcessorManager::AreBuildersUnchanged(AZStd::string_view builderEntries, int& numBuildersEmittingSourceDependencies)
     {
         // each entry here is of the format "builderID~builderFingerprint"

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h

@@ -690,6 +690,11 @@ namespace AssetProcessor
 
         AZStd::unique_ptr<ExcludedFolderCache> m_excludedFolderCache{};
 
+        // Cache of source -> list of dependencies for startup
+        AZStd::unordered_map<AZ::Uuid, AZStd::vector<AzToolsFramework::AssetDatabase::PathOrUuid>> m_dependencyCache;
+        // Cache is turned off after initial idle, it is not meant to handle invalidation or mixed dependency type queries.
+        bool m_dependencyCacheEnabled = true;
+
 protected Q_SLOTS:
         void FinishAnalysis(SourceAssetReference sourceAsset);
         //////////////////////////////////////////////////////////

Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp

@@ -1869,6 +1869,7 @@ TEST_F(AssetProcessorManagerTest, QueryAbsolutePathDependenciesRecursive_BasicTe
 
     // eliminate b --> c
     ASSERT_TRUE(m_assetProcessorManager->m_stateData->RemoveSourceFileDependency(newEntry2.m_sourceDependencyID));
+    m_assetProcessorManager->m_dependencyCache = {};
 
     dependencies.clear();
     m_assetProcessorManager->QueryAbsolutePathDependenciesRecursive(m_aUuid, dependencies, SourceFileDependencyEntry::DEP_SourceToSource);
@@ -1883,6 +1884,9 @@ TEST_F(AssetProcessorManagerTest, QueryAbsolutePathDependenciesRecursive_WithDif
     // test to make sure that different TYPES of dependencies work as expected.
     using namespace AzToolsFramework::AssetDatabase;
 
+    // Cache does not handle mixed dependency types
+    m_assetProcessorManager->m_dependencyCacheEnabled = false;
+
     UnitTestUtils::CreateDummyFile(m_assetRootDir.absoluteFilePath("subfolder1/a.txt"), QString("tempdata\n"));
     UnitTestUtils::CreateDummyFile(m_assetRootDir.absoluteFilePath("subfolder1/b.txt"), QString("tempdata\n"));
     UnitTestUtils::CreateDummyFile(m_assetRootDir.absoluteFilePath("subfolder1/c.txt"), QString("tempdata\n"));
@@ -1995,6 +1999,7 @@ TEST_F(AssetProcessorManagerTest, QueryAbsolutePathDependenciesRecursive_Missing
 
     // eliminate b --> c
     ASSERT_TRUE(m_assetProcessorManager->m_stateData->RemoveSourceFileDependency(newEntry2.m_sourceDependencyID));
+    m_assetProcessorManager->m_dependencyCache = {};
 
     dependencies.clear();
     m_assetProcessorManager->QueryAbsolutePathDependenciesRecursive(m_aUuid, dependencies, SourceFileDependencyEntry::DEP_SourceToSource);
@@ -2405,7 +2410,7 @@ bool SearchDependencies(AzToolsFramework::AssetDatabase::ProductDependencyDataba
 
 void VerifyDependencies(
     AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer dependencyContainer,
-    AZStd::vector<AZ::Data::AssetId> assetIds, 
+    AZStd::vector<AZ::Data::AssetId> assetIds,
     AZStd::initializer_list<const char*> unresolvedPaths = {})
 {
     EXPECT_EQ(dependencyContainer.size(), assetIds.size() + unresolvedPaths.size());
@@ -3997,6 +4002,9 @@ void SourceFileDependenciesTest::SetUp()
     QFile(m_assetRootDir.absoluteFilePath("subfolder1/b.txt")).remove();
     QFile(m_assetRootDir.absoluteFilePath("subfolder1/c.txt")).remove();
     QFile(m_assetRootDir.absoluteFilePath("subfolder1/d.txt")).remove();
+
+    // Cache does not handle mixed dependency types
+    m_assetProcessorManager->m_dependencyCacheEnabled = false;
 }
 
 void SourceFileDependenciesTest::SetupData(
@@ -5111,6 +5119,9 @@ TEST_F(AssetProcessorManagerTest, UpdateSourceFileDependenciesDatabase_WildcardM
     // find existing files which match the dependency and add them as either job or source file dependencies,
     // And recognize matching files as dependencies
 
+    // Cache does not handle mixed dependency types
+    m_assetProcessorManager->m_dependencyCacheEnabled = false;
+
     AZ::Uuid dummyBuilderUUID = AZ::Uuid::CreateRandom();
     UnitTestUtils::CreateDummyFile(m_assetRootDir.absoluteFilePath("subfolder1/wildcardTest.txt"));
     QString relFileName("wildcardTest.txt");

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp

@@ -1606,7 +1606,7 @@ namespace AssetProcessor
     // This function is one of the most frequently called ones in the entire application
     // and is invoked several times per file.  It can frequently become a bottleneck, so
     // avoid doing expensive operations here, especially memory or IO operations.
-    QString PlatformConfiguration::FindFirstMatchingFile(QString relativeName, bool skipIntermediateScanFolder) const
+    QString PlatformConfiguration::FindFirstMatchingFile(QString relativeName, bool skipIntermediateScanFolder, const ScanFolderInfo** outScanFolderInfo) const
     {
         if (relativeName.isEmpty())
         {
@@ -1671,6 +1671,11 @@ namespace AssetProcessor
             {
                 if (fileStateInterface->GetFileInfo(absolutePath, &fileStateInfo))
                 {
+                    if (outScanFolderInfo)
+                    {
+                        *outScanFolderInfo = &scanFolderInfo;
+                    }
+
                     return AssetUtilities::NormalizeFilePath(fileStateInfo.m_absolutePath);
                 }
             }

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h

@@ -242,7 +242,7 @@ namespace AssetProcessor
         QString GetOverridingFile(QString relativeName, QString scanFolderName) const;
 
         //! given a relative name, loop over folders and resolve it to a full path with the first existing match.
-        QString FindFirstMatchingFile(QString relativeName, bool skipIntermediateScanFolder = false) const;
+        QString FindFirstMatchingFile(QString relativeName, bool skipIntermediateScanFolder = false, const AssetProcessor::ScanFolderInfo** scanFolderInfo = nullptr) const;
 
         //! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
         QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false,
@@ -300,7 +300,7 @@ namespace AssetProcessor
         void PopulatePlatformsForScanFolder(AZStd::vector<AssetBuilderSDK::PlatformInfo>& platformsList, QStringList includeTagsList = QStringList(), QStringList excludeTagsList = QStringList());
 
         // uses const + mutability since its a cache.
-        void CacheIntermediateAssetsScanFolderId() const; 
+        void CacheIntermediateAssetsScanFolderId() const;
         AZStd::optional<AZ::s64> GetIntermediateAssetsScanFolderId() const;
         void ReadMetaDataFromSettingsRegistry();