Improves AssetProcessorBatch startup time (#12654)

* Improves AssetProcessorBatch startup time

Also fixes a unit test that was always failing on mac (nobody was
running the unit tests on mac) due to the way temp folders may be
inside a symlink.

I also replaced most scan folders in the config file with a glob
version instead of a regex version, which is significantly faster.

I also eliminated legacy excludes that are either covered by another
exclude (for example the temp hold folder is already covered by the
broader temp exclude) as well as those that just don't exist anymore
such as the animation compression folder (from legacy CryEngine).

This will also improve the Asset Processor GUI version startup time.

This set of changes come from profiling the AP startup time when the
cache is already mostly populated (Zero Analyis Mode).  It optimizes the
functions that were consuming most of the time and uses the file list
gotten from the scanner to skip having to compute the excluded folders
or excluded files more than once.

This was tested by running the AP Batch version with --zeroAnalysisMode
parameter, as well as --project-path parameter to point it at
AutomatedTesting, as well as running the automated tests.

I also ensured that the number of files returned from the scanner
is the same before and after this change, to ensure that the globs
were equivalent to the regees.

Signed-off-by: lawsonamzn <[email protected]>

AutomatedTesting/Gem/AssetProcessorGemConfig.setreg

@@ -3,24 +3,24 @@
         "AssetProcessor": {
             "Settings": {
                 "Exclude PythonTest Benchmark Settings Assets": {
-                    "pattern": "(^|.+/)PythonTests/.*benchmarksettings"
+                    "glob": "*/PythonTests/*.benchmarksettings"
                 },
                 "Exclude fbx_tests": {
-                    "pattern": "(^|.+/)fbx_tests/assets(/.+)$"
+                    "glob": "*/fbx_tests/assets/*"
                 },
                 "Exclude wwise_bank_dependency_tests": {
-                    "pattern": "(^|.+/)wwise_bank_dependency_tests/assets(/.+)$"
+                    "glob": "*/wwise_bank_dependency_tests/assets/*"
                 },
                 "Exclude AssetProcessorTestAssets": {
-                    "pattern": "(^|.+/)asset_processor_tests/assets(/.+)$"
+                    "glob": "*/asset_processor_tests/assets/*"
                 },
                 "Exclude Restricted AssetProcessorTestAssets": {
-                    "pattern": "(^|.+/)asset_processor_tests/restricted(/.+)$"
+                    "glob": "*/asset_processor_tests/restricted/*"
                 },
-                "Exclude Scene_Tests": {
-                    "pattern": "(^|.+/)scene_tests/assets(/.+)$"
-                }  
+                "Exclude Scene Tests": {
+                    "glob": "*/scene_tests/assets/*"
+                }
             }
         }
     }
-}
+}

AutomatedTesting/Gem/Sponza/Registry/AssetProcessorPlatformConfig.setreg

@@ -6,7 +6,7 @@
                 // Sample Gems, Block source folders
                 // ------------------------------------------------------------------------------
                 "Exclude Work In Progress Folders": {
-                    "pattern": "(^|.+/).[Ss]rc(/.*)?$"
+                    "glob": ".src/*"
                 }
             }
         }

Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.cpp

@@ -25,10 +25,51 @@ namespace AssetProcessor
         AZ::Interface<ExcludedFolderCacheInterface>::Unregister(this);
     }
 
+    void ExcludedFolderCache::InitializeFromKnownSet(AZStd::unordered_set<AZStd::string>&& excludedFolders)
+    {
+        m_excludedFolders = AZStd::move(excludedFolders);
+
+        // Add the cache to the list as well
+        AZStd::string projectCacheRootValue;
+        AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
+        projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
+        m_excludedFolders.emplace(projectCacheRootValue);
+
+        // Register to be notified about deletes so we can remove old ignored folders
+        auto fileStateCache = AZ::Interface<IFileStateRequests>::Get();
+
+        if (fileStateCache)
+        {
+            m_handler = AZ::Event<FileStateInfo>::Handler(
+                [this](FileStateInfo fileInfo)
+                {
+                    if (fileInfo.m_isDirectory)
+                    {
+                        AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+
+                        m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData());
+                    }
+                });
+
+            fileStateCache->RegisterForDeleteEvent(m_handler);
+        }
+        else
+        {
+            AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface");
+        }
+
+        m_builtCache = true;
+    }
+
     const AZStd::unordered_set<AZStd::string>& ExcludedFolderCache::GetExcludedFolders()
     {
         if (!m_builtCache)
         {
+            AZ_Warning("AssetProcessor", false, "ExcludedFolderCache lazy-rebuilding instead of being prepopulated (May impact startup performance).  Call InitializeFromKnownSet.\n");
+
+            // manually warm up the cache.
+            AZStd::unordered_set<AZStd::string> excludedFolders;
+
             for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i)
             {
                 const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i);
@@ -36,7 +77,7 @@ namespace AssetProcessor
                 QString absolutePath = rooted.absolutePath();
                 AZStd::stack<QString> dirs;
                 dirs.push(absolutePath);
-                
+
                 while (!dirs.empty())
                 {
                     absolutePath = dirs.top();
@@ -54,7 +95,7 @@ namespace AssetProcessor
                         if (m_platformConfig->IsFileExcluded(pathMatch))
                         {
                             // Add the folder to the list and do not proceed any deeper
-                            m_excludedFolders.emplace(pathMatch.toUtf8().constData());
+                            excludedFolders.emplace(pathMatch.toUtf8().constData());
                         }
                         else if (scanFolderInfo.RecurseSubFolders())
                         {
@@ -65,35 +106,8 @@ namespace AssetProcessor
                 }
             }
 
-            // Add the cache to the list as well
-            AZStd::string projectCacheRootValue;
-            AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
-            projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
-            m_excludedFolders.emplace(projectCacheRootValue);
-
-            // Register to be notified about deletes so we can remove old ignored folders
-            auto fileStateCache = AZ::Interface<IFileStateRequests>::Get();
-
-            if (fileStateCache)
-            {
-                m_handler = AZ::Event<FileStateInfo>::Handler([this](FileStateInfo fileInfo)
-                {
-                    if (fileInfo.m_isDirectory)
-                    {
-                        AZStd::scoped_lock lock(m_pendingNewFolderMutex);
-
-                        m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData());
-                    }
-                });
-
-                fileStateCache->RegisterForDeleteEvent(m_handler);
-            }
-            else
-            {
-                AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface");
-            }
-
-            m_builtCache = true;
+            InitializeFromKnownSet(AZStd::move(excludedFolders));
+            return m_excludedFolders;
         }
 
         // Incorporate any pending folders

Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.h

@@ -26,6 +26,10 @@ namespace AssetProcessor
 
         void FileAdded(QString path) override;
 
+        //! Initialize the cache from a known list of excluded folders, so that it does not have to do a scan
+        //! for itself.  Destroys the input.
+        void InitializeFromKnownSet(AZStd::unordered_set<AZStd::string>&& excludedFolders);
+
     private:
         bool m_builtCache = false;
         const PlatformConfiguration* m_platformConfig{};

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp

@@ -1558,8 +1558,8 @@ namespace AssetProcessor
                 {
                     // Now that we've verified that the output doesn't conflict with an existing source
                     // And we've updated the database, trigger processing the output
-
-                    AssessFileInternal(productPath.GetIntermediatePath().c_str(), false);
+                    Q_EMIT IntermediateAssetCreated(QString::fromUtf8(productPath.GetIntermediatePath().c_str()));
+                    AssessFileInternal(QString::fromUtf8(productPath.GetIntermediatePath().c_str()), false);
                 }
             }
 
@@ -2656,8 +2656,19 @@ namespace AssetProcessor
             return;
         }
 
+        // During unit tests, it can be the case that cache folders are actually in a temp folder structure
+        // on OSX this is /var/... , but that is a symlink for real path /private/var.  In some circumstances file monitor
+        // for deletions may report the canonical path (/private/var/...) when the 'cache root' or watched folder
+        // is actually /var/...
+        // to account for this, we check if the canonical path to the cache root is not the same as the path we are running at
+        // if they are different, incoming files may need a fixup that involves replacing their canonical paths (de-symlinked)
+        // with the paths we expect (symlinked).
+        
         QString canonicalRootDir = AssetUtilities::NormalizeFilePath(m_cacheRootDir.canonicalPath());
-
+        
+        // we only need to do a fixup if the root dir is not the same as the canonical root dir.
+        bool needCanonicalFixup = canonicalRootDir != m_cacheRootDir.canonicalPath();
+        
         FileExamineContainer swapped;
         m_filesToExamine.swap(swapped); // makes it okay to call CheckSource(...)
 
@@ -2685,30 +2696,13 @@ namespace AssetProcessor
 
             // examination occurs here.
             // first, is it a source or is it a product in the cache folder?
-
-
-            QString normalizedPath = examineFile.m_fileName.toUtf8().constData();
-
-            AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessFilesToExamineQueue: %s delete: %s.\n", examineFile.m_fileName.toUtf8().constData(), examineFile.m_isDelete ? "true" : "false");
-
-            // debug-only check to make sure our assumption about normalization is correct.
+            QString normalizedPath = examineFile.m_fileName;
+             // debug-only check to make sure our assumption about normalization is correct.
             Q_ASSERT(normalizedPath == AssetUtilities::NormalizeFilePath(normalizedPath));
-
-            // if its in the cache root then its a product file:
-            bool isProductFile = IsInCacheFolder(examineFile.m_fileName.toUtf8().constData());
-#if AZ_TRAIT_OS_PLATFORM_APPLE
-            // a case can occur on apple platforms in the temp folders
-            // where there is a symlink and /var/folders/.../ is also known
-            // as just /private/var/folders/...
-            // this tends to happen for delete notifies and we can't canonicalize incoming delete notifies
-            // because the file has already been deleted and thus its canonical path cannot be found.  Instead
-            // we will use the canonical path of the cache root dir instead, and then alter the file
-            // to have the current cache root dir instead.
-            if ((!isProductFile)&&(!canonicalRootDir.isEmpty()))
-            {
-                // try the canonicalized form:
-                isProductFile = examineFile.m_fileName.startsWith(canonicalRootDir);
-                if (isProductFile)
+            
+            if (needCanonicalFixup)
+            {
+                if (normalizedPath.startsWith(canonicalRootDir))
                 {
                     // found in canonical location, update its normalized path
                     QString withoutCachePath = normalizedPath.mid(canonicalRootDir.length() + 1);
@@ -2716,7 +2710,12 @@ namespace AssetProcessor
                     normalizedPath = AssetUtilities::NormalizeFilePath(m_cacheRootDir.absoluteFilePath(withoutCachePath));
                 }
             }
-#endif // AZ_TRAIT_OS_PLATFORM_APPLE
+
+            AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessFilesToExamineQueue: %s delete: %s.\n", examineFile.m_fileName.toUtf8().constData(), examineFile.m_isDelete ? "true" : "false");
+
+            // if its in the cache root then its a product file:
+            bool isProductFile = IsInCacheFolder(normalizedPath.toUtf8().constData());
+
             // strip the engine off it so that its a "normalized asset path" with appropriate slashes and such:
             if (isProductFile)
             {
@@ -2966,7 +2965,7 @@ namespace AssetProcessor
 
                 // its an input file or a file we don't care about...
                 // note that if the file now exists, we have to treat it as an input asset even if it came in as a delete.
-                if (examineFile.m_isDelete && !QFile::exists(examineFile.m_fileName))
+                if (examineFile.m_isDelete && !QFile::exists(normalizedPath))
                 {
                     AZ_TracePrintf(AssetProcessor::DebugChannel, "Input was deleted and no overrider was found.\n");
 
@@ -3065,6 +3064,11 @@ namespace AssetProcessor
     // ----------------------------------------------------
     // ------------- File change Queue --------------------
     // ----------------------------------------------------
+
+    // note that this function is on the "hot path" of file analysis
+    // during startup, and should avoid logging, sleeping, or doing
+    // any more work than is necessary (Log only in error or uncommon
+    // circumstances).
     void AssetProcessorManager::AssessFileInternal(QString fullFile, bool isDelete, bool fromScanner)
     {
         if (m_quitRequested)
@@ -3116,8 +3120,6 @@ namespace AssetProcessor
         m_AssetProcessorIsBusy = true;
         Q_EMIT AssetProcessorManagerIdleState(false);
 
-        AZ_TracePrintf(AssetProcessor::DebugChannel, "AssesFileInternal: %s %s\n", normalizedFullFile.toUtf8().constData(), isDelete ? "true" : "false");
-
         // this function is the raw function that gets called from the file monitor
         // whenever an asset has been modified or added (not deleted)
         // it should place the asset on a grace period list and not considered until changes stop happening to it.
@@ -3334,6 +3336,20 @@ namespace AssetProcessor
         }
     }
 
+    // warm up the excluded folder cache with the data from the scanner so that it is not necessary to rescan.
+    void AssetProcessorManager::RecordExcludesFromScanner(QSet<AssetFileInfo> excludePaths)
+    {
+        AZStd::unordered_set<AZStd::string> excludedFolders;
+        for (const auto& folder : excludePaths)
+        {
+            if (folder.m_isDirectory)
+            {
+                excludedFolders.insert(folder.m_filePath.toUtf8().constData());
+            }
+        }
+        m_excludedFolderCache->InitializeFromKnownSet(AZStd::move(excludedFolders));
+    }
+
     bool AssetProcessorManager::CanSkipProcessingFile(const AssetFileInfo &fileInfo, AZ::u64& fileHashOut)
     {
         // Check to see if the file has changed since the last time we saw it

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h

@@ -273,6 +273,8 @@ namespace AssetProcessor
         void JobProcessDurationChanged(JobEntry jobEntry, int durationMs);
         void CreateJobsDurationChanged(QString sourceName);
 
+        void IntermediateAssetCreated(QString newFileAbsolutePath);
+
         //! Send a message when a new path dependency is resolved, so that downstream tools know the AssetId of the resolved dependency.
         void PathDependencyResolved(const AZ::Data::AssetId& assetId, const AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry);
 
@@ -291,6 +293,7 @@ namespace AssetProcessor
 
         void AssessFilesFromScanner(QSet<AssetFileInfo> filePaths);
         void RecordFoldersFromScanner(QSet<AssetFileInfo> folderPaths);
+        void RecordExcludesFromScanner(QSet<AssetFileInfo> excludePaths);
 
         virtual void AssessModifiedFile(QString filePath);
         virtual void AssessAddedFile(QString filePath);

Code/Tools/AssetProcessor/native/AssetManager/assetScannerWorker.cpp

@@ -9,6 +9,7 @@
 #include "native/AssetManager/assetScanner.h"
 #include "native/utilities/PlatformConfiguration.h"
 #include <QDir>
+#include <QtConcurrent/QtConcurrentFilter>
 
 using namespace AssetProcessor;
 
@@ -25,6 +26,8 @@ void AssetScannerWorker::StartScan()
 
     m_fileList.clear();
     m_folderList.clear();
+    m_excludedList.clear();
+    
     m_doScan = true;
 
     AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Scanning file system for changes...\n");
@@ -46,6 +49,8 @@ void AssetScannerWorker::StartScan()
     {
         m_fileList.clear();
         m_folderList.clear();
+        m_excludedList.clear();
+        
         Q_EMIT ScanningStateChanged(AssetProcessor::AssetScanningStatus::Stopped);
         return;
     }
@@ -74,64 +79,118 @@ void AssetScannerWorker::ScanForSourceFiles(const ScanFolderInfo& scanFolderInfo
         return;
     }
 
-    QDir dir(scanFolderInfo.ScanPath());
-
     QFileInfoList entries;
 
-    //Only scan sub folders if recurseSubFolders flag is set
-    if (!scanFolderInfo.RecurseSubFolders())
-    {
-        entries = dir.entryInfoList(QDir::NoDotAndDotDot | QDir::Files);
-    }
-    else
-    {
-        entries = dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::Files);
-    }
+    QDir cacheDir;
+    AssetUtilities::ComputeProjectCacheRoot(cacheDir);
+    QString normalizedCachePath = AssetUtilities::NormalizeDirectoryPath(cacheDir.absolutePath());
+    AZ::IO::Path cachePath(normalizedCachePath.toUtf8().constData());
 
-    for (const QFileInfo& entry : entries)
-    {
-        if (!m_doScan) // scan was cancelled!
-        {
-            return;
-        }
+    QString intermediateAssetsFolder = QString::fromUtf8(AssetUtilities::GetIntermediateAssetsFolder(cachePath).c_str());
+    QString normalizedIntermediateAssetsFolder = AssetUtilities::NormalizeDirectoryPath(intermediateAssetsFolder);
 
-        QString absPath = entry.absoluteFilePath();
-        const bool isDirectory = entry.isDir();
-        QDateTime modTime = entry.lastModified();
-        AZ::u64 fileSize = isDirectory ? 0 : entry.size();
-        AssetFileInfo assetFileInfo(absPath, modTime, fileSize, &rootScanFolder, isDirectory);
+    // Implemented non-recursively so that the above functions only have to be called once per scan,
+    // and so that the performance is easy to analyze in a profiler:
+    QList<ScanFolderInfo> pathsToScanQueue;
+    pathsToScanQueue.push_back(scanFolderInfo);
 
-        // Filtering out excluded files
-        if (m_platformConfiguration->IsFileExcluded(absPath))
+    while (!pathsToScanQueue.empty())
+    {
+        ScanFolderInfo pathToScan = pathsToScanQueue.back();
+        pathsToScanQueue.pop_back();
+        QDir dir(pathToScan.ScanPath());
+        dir.setSorting(QDir::Unsorted);
+        // Only scan sub folders if recurseSubFolders flag is set
+        if (!scanFolderInfo.RecurseSubFolders())
         {
-            m_excludedList.insert(AZStd::move(assetFileInfo));
-            continue;
+            entries = dir.entryInfoList(QDir::NoDotAndDotDot | QDir::Files);
         }
-
-        if (isDirectory)
+        else
         {
-            //Entry is a directory
-            // The AP needs to know about all directories so it knows when a delete occurs if the path refers to a folder or a file
-            m_folderList.insert(AZStd::move(assetFileInfo));
-            ScanFolderInfo tempScanFolderInfo(absPath, "", "", false, true);
-            ScanForSourceFiles(tempScanFolderInfo, rootScanFolder);
+            entries = dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::Files);
         }
-        else if (!AssetUtilities::IsInCacheFolder(absPath.toUtf8().constData())) // Ignore files in the cache
+
+        for (const QFileInfo& entry : entries)
         {
-            //Entry is a file
-            m_fileList.insert(AZStd::move(assetFileInfo));
+            if (!m_doScan) // scan was cancelled!
+            {
+                return;
+            }
+
+            QString absPath = entry.absoluteFilePath();
+            const bool isDirectory = entry.isDir();
+            QDateTime modTime = entry.lastModified();
+            AZ::u64 fileSize = isDirectory ? 0 : entry.size();
+            AssetFileInfo assetFileInfo(absPath, modTime, fileSize, &rootScanFolder, isDirectory);
+            QString relPath = absPath.mid(rootScanFolder.ScanPath().length() + 1);
+
+            if (isDirectory)
+            {
+                // in debug, assert that the paths coming from qt directory info iteration is already normalized
+                // allowing us to skip normalization and know that comparisons like "IsInCacheFolder" will actually succed.
+                Q_ASSERT(absPath == AssetUtilities::NormalizeDirectoryPath(absPath));
+                // Filtering out excluded directories immediately (not in a thread pool) since that prevents us from recursing.
+
+                // we already know the root scan folder, and can thus chop that part off and call the cheaper IsFileExcludedRelPath:
+
+                if (m_platformConfiguration->IsFileExcludedRelPath(relPath))
+                {
+                    m_excludedList.insert(AZStd::move(assetFileInfo));
+                    continue;
+                }
+
+                // Entry is a directory
+                // The AP needs to know about all directories so it knows when a delete occurs if the path refers to a folder or a file
+                m_folderList.insert(AZStd::move(assetFileInfo));
+
+                // recurse into this folder.
+                // Since we only care about source files, we can skip cache folders that are not the Intermediate Assets Folder.
+
+                if (absPath.startsWith(normalizedCachePath))
+                {
+                    // its in the cache.  Is it the cache itself?
+                    if (absPath.length() != normalizedCachePath.length())
+                    {
+                        // no.  Is it in the intermediateassets?
+                        if (!absPath.startsWith(normalizedIntermediateAssetsFolder))
+                        {
+                            // Its not something in the intermediate assets folder, nor is it the cache itself,
+                            // so it is just a file somewhere in the cache.
+                            continue; // do not recurse.
+                        }
+                    }
+                }
+                // then we can recurse.  Otherwise, its a non-intermediate-assets-folder
+                ScanFolderInfo tempScanFolderInfo(absPath, "", "", false, true);
+                pathsToScanQueue.push_back(tempScanFolderInfo);
+            }
+            else
+            {
+                // Entry is a file
+                Q_ASSERT(absPath == AssetUtilities::NormalizeFilePath(absPath));
+
+                if (!AssetUtilities::IsInCacheFolder(absPath.toUtf8().constData(), cachePath)) // Ignore files in the cache
+                {
+                    if (!m_platformConfiguration->IsFileExcludedRelPath(relPath))
+                    {
+                        m_fileList.insert(AZStd::move(assetFileInfo));
+                    }
+                    else
+                    {
+                        m_excludedList.insert(AZStd::move(assetFileInfo));
+                    }
+                }
+            }
         }
     }
 }
 
 void AssetScannerWorker::EmitFiles()
 {
-    //Loop over all source asset files and send them up the chain:
     Q_EMIT FilesFound(m_fileList);
     m_fileList.clear();
     Q_EMIT FoldersFound(m_folderList);
     m_folderList.clear();
     Q_EMIT ExcludedFound(m_excludedList);
     m_excludedList.clear();
-
 }

Code/Tools/AssetProcessor/native/AssetManager/assetScannerWorker.h

@@ -53,6 +53,7 @@ Q_SIGNALS:
         QSet<AssetFileInfo> m_fileList; // note:  neither QSet nor QString are qobject-derived
         QSet<AssetFileInfo> m_folderList;
         QSet<AssetFileInfo> m_excludedList;
+
         PlatformConfiguration* m_platformConfiguration;
     };
 } // end namespace AssetProcessor

Code/Tools/AssetProcessor/native/FileProcessor/FileProcessor.cpp

@@ -184,16 +184,16 @@ namespace AssetProcessor
         for (const AssetFileInfo& fileInfo : m_filesInAssetScanner)
         {
             bool isDir = fileInfo.m_isDirectory;
-            QString scanFolderName;
+            QString scanFolderPath;
             QString relativeFileName;
 
-            if (!m_platformConfig->ConvertToRelativePath(fileInfo.m_filePath, relativeFileName, scanFolderName))
+            if (!m_platformConfig->ConvertToRelativePath(fileInfo.m_filePath, relativeFileName, scanFolderPath))
             {
                 AZ_Error(AssetProcessor::ConsoleChannel, false, "Failed to convert full path to relative for file %s", fileInfo.m_filePath.toUtf8().constData());
                 continue;
             }
 
-            const ScanFolderInfo* scanFolderInfo = m_platformConfig->GetScanFolderForFile(fileInfo.m_filePath);
+            const ScanFolderInfo* scanFolderInfo = m_platformConfig->GetScanFolderByPath(scanFolderPath);
             if (!scanFolderInfo)
             {
                 AZ_Error(AssetProcessor::ConsoleChannel, false, "Failed to find the scan folder for file %s", fileInfo.m_filePath.toUtf8().constData());

Code/Tools/AssetProcessor/native/tests/ApplicationManagerTests.cpp

@@ -77,26 +77,7 @@ namespace UnitTests
 
     using BatchApplicationManagerTest = UnitTest::LeakDetectionFixture;
 
-    TEST_F(BatchApplicationManagerTest, FileCreatedOnDisk_ShowsUpInFileCache)
-    {
-        AssetProcessor::MockAssetDatabaseRequestsHandler m_databaseLocationListener;
-        AZ::IO::Path assetRootDir(m_databaseLocationListener.GetAssetRootDir());
-
-        int argc = 0;
-
-        auto m_applicationManager = AZStd::make_unique<MockBatchApplicationManager>(&argc, nullptr);
-        m_applicationManager->InitFileStateCache();
-
-        auto* fileStateCache = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
-
-        ASSERT_TRUE(fileStateCache);
-
-        EXPECT_FALSE(fileStateCache->Exists((assetRootDir / "test").c_str()));
-        UnitTestUtils::CreateDummyFile((assetRootDir / "test").c_str());
-        EXPECT_TRUE(fileStateCache->Exists((assetRootDir / "test").c_str()));
-    }
-
-    TEST_F(ApplicationManagerTest, FileWatcherEventsTriggered_ProperlySignalledOnCorrectThread_SUITE_sandbox)
+    TEST_F(ApplicationManagerTest, FileWatcherEventsTriggered_ProperlySignalledOnCorrectThread)
     {
         AZ::IO::Path assetRootDir(m_databaseLocationListener.GetAssetRootDir());
 

Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp

@@ -8,6 +8,8 @@
 
 #include "AssetProcessorManagerTest.h"
 #include "native/AssetManager/PathDependencyManager.h"
+#include "native/AssetManager/assetScannerWorker.h"
+
 #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzToolsFramework/Asset/AssetProcessorMessages.h>
 #include <AzToolsFramework/ToolsFileUtils/ToolsFileUtils.h>
@@ -4880,7 +4882,53 @@ TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache)
     ASSERT_TRUE(excludedFolderCacheInterface);
 
     {
+        m_errorAbsorber->Clear();
+        const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+        m_errorAbsorber->ExpectWarnings(1); // because we didn't precache, we'd expect a warning here, about performance.
+
+        ASSERT_EQ(excludedFolders.size(), 3);
+    }
+
+    m_fileStateCache->SignalDeleteEvent(m_assetRootDir.absoluteFilePath("subfolder2/folder/two/ignored"));
+
+    const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+
+    ASSERT_EQ(excludedFolders.size(), 2);
+}
+
+// same as above test but actually runs a file scanner over the root dir and ensures it still functions
+TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache_WithPrecache)
+{
+    // Add a file to a new ignored folder
+    QString newFilePath = m_assetRootDir.absoluteFilePath("subfolder2/folder/two/ignored/three/new.foo");
+    UnitTestUtils::CreateDummyFile(newFilePath);
+
+    {
+        // warm up the cache.
+        AssetScannerWorker worker(m_config.get());
+        bool foundExcludes = false;
+        QObject::connect(
+            &worker,
+            &AssetScannerWorker::ExcludedFound,
+            m_assetProcessorManager.get(),
+            [&](QSet<AssetFileInfo> excluded)
+            {
+                foundExcludes = true;
+                m_assetProcessorManager->RecordExcludesFromScanner(excluded);
+            });
+
+        worker.StartScan();
+        QCoreApplication::processEvents();
+        ASSERT_TRUE(foundExcludes);
+    }
+
+    auto excludedFolderCacheInterface = AZ::Interface<ExcludedFolderCacheInterface>::Get();
+    ASSERT_TRUE(excludedFolderCacheInterface);
+
+    {
+        m_errorAbsorber->Clear();
         const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+        m_errorAbsorber->ExpectWarnings(0);  // we precached, so there should not be a warning.
 
         ASSERT_EQ(excludedFolders.size(), 3);
     }

Code/Tools/AssetProcessor/native/ui/MainWindow.cpp

@@ -388,7 +388,6 @@ void MainWindow::Activate()
     // Asset view
     m_sourceAssetTreeFilterModel = new AssetProcessor::SourceAssetTreeFilterModel(this);
     m_sourceModel = new AssetProcessor::SourceAssetTreeModel(m_sharedDbConnection, this);
-    m_sourceModel->Reset();
     m_sourceAssetTreeFilterModel->setSourceModel(m_sourceModel);
     ui->SourceAssetsTreeView->setModel(m_sourceAssetTreeFilterModel);
     ui->SourceAssetsTreeView->setColumnWidth(aznumeric_cast<int>(AssetTreeColumns::Extension), 80);
@@ -399,7 +398,6 @@ void MainWindow::Activate()
     m_intermediateAssetTreeFilterModel = new AssetProcessor::AssetTreeFilterModel(this);
     m_intermediateModel = new AssetProcessor::SourceAssetTreeModel(m_sharedDbConnection, this);
     m_intermediateModel->SetOnlyShowIntermediateAssets();
-    m_intermediateModel->Reset();
     m_intermediateAssetTreeFilterModel->setSourceModel(m_intermediateModel);
     ui->IntermediateAssetsTreeView->setModel(m_intermediateAssetTreeFilterModel);
     connect(
@@ -411,7 +409,6 @@ void MainWindow::Activate()
 
     m_productAssetTreeFilterModel = new AssetProcessor::AssetTreeFilterModel(this);
     m_productModel = new AssetProcessor::ProductAssetTreeModel(m_sharedDbConnection, this);
-    m_productModel->Reset();
     m_productAssetTreeFilterModel->setSourceModel(m_productModel);
     ui->ProductAssetsTreeView->setModel(m_productAssetTreeFilterModel);
     ui->ProductAssetsTreeView->setColumnWidth(aznumeric_cast<int>(AssetTreeColumns::Extension), 80);
@@ -493,6 +490,23 @@ void MainWindow::Activate()
         &MainWindow::ShowIncomingProductDependenciesContextMenu);
 
     SetupAssetSelectionCaching();
+    // the first time we open that panel we can refresh it.
+    m_connectionForResettingAssetsView = connect(
+        ui->dialogStack,
+        &QStackedWidget::currentChanged,
+        this,
+        [&](int index)
+        {
+            if (index == static_cast<int>(DialogStackIndex::Assets))
+            {
+                // the first time we show the asset window, reset the model since its so expensive to do on every startup
+                // and many times, the user does not even go to that panel.
+                m_sourceModel->Reset();
+                m_intermediateModel->Reset();
+                m_productModel->Reset();
+                QObject::disconnect(m_connectionForResettingAssetsView);
+            }
+        });
 
     //Log View
     m_loggingPanel = ui->LoggingPanel;

Code/Tools/AssetProcessor/native/ui/MainWindow.h

@@ -256,5 +256,6 @@ private:
 
     AZStd::string m_cachedSourceAssetSelection;
     AZStd::string m_cachedProductAssetSelection;
+    QMetaObject::Connection m_connectionForResettingAssetsView;
 };
 

Code/Tools/AssetProcessor/native/ui/ProductAssetTreeModel.cpp

@@ -61,6 +61,12 @@ namespace AssetProcessor
             return;
         }
 
+        if (!m_root)
+        {
+            // no need to update the model if the root hasn't been created via ResetModel()
+            return;
+        }
+
         // Model changes need to be run on the main thread.
         AZ::SystemTickBus::QueueFunction([&, entry]()
         {
@@ -128,6 +134,12 @@ namespace AssetProcessor
             return;
         }
 
+        if (!m_root)
+        {
+            // we haven't reset the model yet, which means all of this will happen when we do.
+            return;
+        }
+
         // UI changes need to be done on the main thread.
         AZ::SystemTickBus::QueueFunction([&, productId]()
         {
@@ -142,6 +154,12 @@ namespace AssetProcessor
             return;
         }
 
+        if (!m_root)
+        {
+            // we haven't reset the model yet, which means all of this will happen when we do.
+            return;
+        }
+
         // UI changes need to be done on the main thread.
         AZ::SystemTickBus::QueueFunction([&, products]()
         {

Code/Tools/AssetProcessor/native/ui/SourceAssetTreeModel.cpp

@@ -244,6 +244,12 @@ namespace AssetProcessor
             return;
         }
 
+        if (!m_root)
+        {
+            // we haven't reset the model yet, which means all of this will happen when we do.
+            return;
+        }
+
         // Model changes need to be run on the main thread.
         AZ::SystemTickBus::QueueFunction([&, entry]()
             {
@@ -304,6 +310,12 @@ namespace AssetProcessor
             return;
         }
 
+        if (!m_root)
+        {
+            // we haven't reset the model yet, which means all of this will happen when we do.
+            return;
+        }
+
         // UI changes need to be done on the main thread.
         AZ::SystemTickBus::QueueFunction([&, sourceId]()
             {

Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp

@@ -367,7 +367,9 @@ void ApplicationManagerBase::InitAssetScanner()
     // asset processor manager
     QObject::connect(m_assetScanner, &AssetScanner::AssetScanningStatusChanged, m_assetProcessorManager, &AssetProcessorManager::OnAssetScannerStatusChange);
     QObject::connect(m_assetScanner, &AssetScanner::FilesFound,                 m_assetProcessorManager, &AssetProcessorManager::AssessFilesFromScanner);
-    QObject::connect(m_assetScanner, &AssetScanner::FoldersFound,                 m_assetProcessorManager, &AssetProcessorManager::RecordFoldersFromScanner);
+    QObject::connect(m_assetScanner, &AssetScanner::FoldersFound,               m_assetProcessorManager, &AssetProcessorManager::RecordFoldersFromScanner);
+    QObject::connect(m_assetScanner, &AssetScanner::ExcludedFound,              m_assetProcessorManager, &AssetProcessorManager::RecordExcludesFromScanner);
+
 
     QObject::connect(m_assetScanner, &AssetScanner::FilesFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
     QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
@@ -1451,8 +1453,39 @@ bool ApplicationManagerBase::Activate()
 
     InitFileStateCache();
     InitFileProcessor();
-
     InitUuidManager();
+
+    // now that apm, statecache, processor, and uuid manager are all alive, hook them up to the signal that AP
+    // gives when it modifies an intermediate asset.  For the file cache, we hook it up directly so that there is
+    // no delay between the notification and the invalidation/creation of its cache entry.
+
+    auto notifyFileStateCache = [this](QString changedFile)
+    {
+        // the file state cache will get this immediately and inline, it needs to treat it with thread safety.
+        m_fileStateCache->UpdateFile(changedFile);
+    };
+
+    auto notifyUuidManagerAndFileProcessor = [this](QString changedFile)
+    {
+        // these are not necessarily time sensitive.
+        m_uuidManager->FileChanged(changedFile.toUtf8().constData());
+        m_fileProcessor->AssessAddedFile(changedFile);
+    };
+         
+    QObject::connect(
+        m_assetProcessorManager,
+        &AssetProcessor::AssetProcessorManager::IntermediateAssetCreated,
+        this,
+        notifyFileStateCache,
+        Qt::DirectConnection);
+
+    QObject::connect(
+        m_assetProcessorManager,
+        &AssetProcessor::AssetProcessorManager::IntermediateAssetCreated,
+        this,
+        notifyUuidManagerAndFileProcessor,
+        Qt::QueuedConnection);
+
     InitAssetCatalog();
     InitFileMonitor(AZStd::make_unique<FileWatcher>());
     InitAssetScanner();

Code/Tools/AssetProcessor/native/utilities/BatchApplicationManager.cpp

@@ -107,15 +107,6 @@ void BatchApplicationManager::InitSourceControl()
     }
 }
 
-void BatchApplicationManager::InitFileStateCache()
-{
-    // File state cache is disabled for batch mode because it relies on the file monitor to function properly.
-    // Since the file monitor is disabled, the cache must be disabled as well.
-    // Note the main reason this is disabled is because intermediate assets can be created during processing which would
-    // need to be recorded in the cache.
-    m_fileStateCache = AZStd::make_unique<AssetProcessor::FileStatePassthrough>();
-}
-
 void BatchApplicationManager::MakeActivationConnections()
 {
     QObject::connect(m_rcController, &AssetProcessor::RCController::FileCompiled,

Code/Tools/AssetProcessor/native/utilities/BatchApplicationManager.h

@@ -40,7 +40,6 @@ private:
     const char* GetLogBaseName() override;
     RegistryCheckInstructions PopupRegistryProblemsMessage(QString warningText) override;
     void InitSourceControl() override;
-    void InitFileStateCache() override;
 
     void MakeActivationConnections() override;
 

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp

@@ -977,7 +977,7 @@ namespace AssetProcessor
         }
     }
 
-    void PlatformConfiguration::CacheIntermediateAssetsScanFolderId()
+    void PlatformConfiguration::CacheIntermediateAssetsScanFolderId() const
     {
         for (const auto& scanfolder : m_scanFolders)
         {
@@ -1241,8 +1241,11 @@ namespace AssetProcessor
 
                 // New assets can be saved in any scan folder defined except for the engine root.
                 const bool canSaveNewAssets = !isEngineRoot;
+
+                QString watchFolderPath = QString::fromUtf8(scanFolderEntry.m_watchPath.c_str(), static_cast<int>(scanFolderEntry.m_watchPath.Native().size()));
+                watchFolderPath = AssetUtilities::NormalizeDirectoryPath(watchFolderPath);
                 AddScanFolder(ScanFolderInfo(
-                    QString::fromUtf8(scanFolderEntry.m_watchPath.c_str(), aznumeric_cast<int>(scanFolderEntry.m_watchPath.Native().size())),
+                    watchFolderPath,
                     QString::fromUtf8(scanFolderEntry.m_scanFolderDisplayName.c_str(), aznumeric_cast<int>(scanFolderEntry.m_scanFolderDisplayName.size())),
                     QString::fromUtf8(scanFolderEntry.m_scanFolderIdentifier.c_str(), aznumeric_cast<int>(scanFolderEntry.m_scanFolderIdentifier.size())),
                     isEngineRoot,
@@ -1462,6 +1465,12 @@ namespace AssetProcessor
         {
             //using a bool instead of using #define UNIT_TEST because the user can also run batch processing in unittest
             m_scanFolders.push_back(source);
+
+            // since we're synthesizing folder adds, assign ascending folder ids if not provided.
+            if (source.ScanFolderID() == 0)
+            {
+                m_scanFolders.back().SetScanFolderID(m_scanFolders.size() - 1);
+            }
             return;
         }
 
@@ -1573,6 +1582,9 @@ namespace AssetProcessor
         return QString();
     }
 
+    // This function is one of the most frequently called ones in the entire application
+    // and is invoked several times per file.  It can frequently become a bottleneck, so
+    // avoid doing expensive operations here, especially memory or IO operations.
     QString PlatformConfiguration::FindFirstMatchingFile(QString relativeName, bool skipIntermediateScanFolder) const
     {
         if (relativeName.isEmpty())
@@ -1582,29 +1594,56 @@ namespace AssetProcessor
 
         auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
 
-        QDir cacheRoot;
-        AssetUtilities::ComputeProjectCacheRoot(cacheRoot);
+        // Only compute the intermediate assets folder path if we are going to search for and skip it.
+       
+        if (skipIntermediateScanFolder)
+        {
+            if (m_intermediateAssetScanFolderId == -1)
+            {
+                CacheIntermediateAssetsScanFolderId();
+            }
+        }
+        
+        QString absolutePath; // avoid allocating memory repeatedly here by reusing absolutePath each scan folder.
+        absolutePath.reserve(AZ_MAX_PATH_LEN);
 
+        QFileInfo details(relativeName); // note that this does not actually hit the actual storage medium until you query something
+        bool isAbsolute = details.isAbsolute(); // note that this looks at the file name string only, it does not hit storage.
+        
         for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
         {
-            AssetProcessor::ScanFolderInfo scanFolderInfo = m_scanFolders[pathIdx];
+            const AssetProcessor::ScanFolderInfo& scanFolderInfo = m_scanFolders[pathIdx];
 
-            if (skipIntermediateScanFolder && AssetUtilities::GetIntermediateAssetsFolder(cacheRoot.absolutePath().toUtf8().constData()) == AZ::IO::PathView(scanFolderInfo.ScanPath().toUtf8().constData()))
+            if ((skipIntermediateScanFolder) && (scanFolderInfo.ScanFolderID() == m_intermediateAssetScanFolderId))
             {
                 // There's only 1 intermediate assets folder, if we've skipped it, theres no point continuing to check every folder afterwards
                 skipIntermediateScanFolder = false;
                 continue;
             }
 
-            QString tempRelativeName(relativeName);
-
-            if ((!scanFolderInfo.RecurseSubFolders()) && (tempRelativeName.contains('/')))
+            if ((!scanFolderInfo.RecurseSubFolders()) && (relativeName.contains('/')))
             {
                 // the name is a deeper relative path, but we don't recurse this scan folder, so it can't win
                 continue;
             }
-            QDir rooted(scanFolderInfo.ScanPath());
-            QString absolutePath = rooted.absoluteFilePath(tempRelativeName);
+
+            if (isAbsolute)
+            {
+                if (!relativeName.startsWith(scanFolderInfo.ScanPath()))
+                {
+                    continue; // its not this scanfolder.
+                }
+                absolutePath = relativeName;
+            }
+            else
+            {
+                // scanfolders are always absolute paths and already normalized.  We can just concatenate.
+                // Do so with minimal allocation by using resize/append, instead of operator+
+                absolutePath.resize(0);
+                absolutePath.append(scanFolderInfo.ScanPath());
+                absolutePath.append('/');
+                absolutePath.append(relativeName);
+            }
             AssetProcessor::FileStateInfo fileStateInfo;
 
             if (fileStateInterface)
@@ -1763,10 +1802,9 @@ namespace AssetProcessor
     //! Given a scan folder path, get its complete info
     const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderByPath(const QString& scanFolderPath) const
     {
-        AZ::IO::Path scanFolderPathView(scanFolderPath.toUtf8().constData());
         for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
         {
-            if (AZ::IO::PathView(m_scanFolders[pathIdx].ScanPath().toUtf8().constData()) == scanFolderPathView)
+            if (m_scanFolders[pathIdx].ScanPath() == scanFolderPath)
             {
                 return &m_scanFolders[pathIdx];
             }
@@ -1801,6 +1839,8 @@ namespace AssetProcessor
         AZStd::vector<AssetBuilderSDK::PlatformInfo> platforms;
         PopulatePlatformsForScanFolder(platforms);
 
+        scanfolderPath = AssetUtilities::NormalizeDirectoryPath(QString::fromUtf8(scanfolderPath.c_str())).toUtf8().constData();
+
         // By default the project scanfolder is recursive with an order of 0
         // The intermediate assets folder needs to be higher priority since its a subfolder (otherwise GetScanFolderForFile won't pick the right scanfolder)
         constexpr int order = -1;
@@ -1935,12 +1975,20 @@ namespace AssetProcessor
         QString relPath, scanFolderName;
         if (ConvertToRelativePath(fileName, relPath, scanFolderName))
         {
-            for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
+            return IsFileExcludedRelPath(relPath);
+        }
+
+        return false;
+    }
+
+    bool AssetProcessor::PlatformConfiguration::IsFileExcludedRelPath(QString relPath) const
+    {
+        AZ::IO::FixedMaxPathString encoded = relPath.toUtf8().constData();
+        for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
+        {
+            if (excludeRecognizer.m_patternMatcher.MatchesPath(encoded.c_str()))
             {
-                if (excludeRecognizer.m_patternMatcher.MatchesPath(relPath.toUtf8().constData()))
-                {
-                    return true;
-                }
+                return true;
             }
         }
 

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h

@@ -222,8 +222,10 @@ namespace AssetProcessor
         void AddMetaDataType(const QString& type, const QString& originalExtension);
 
         // ------------------- utility functions --------------------
-        ///! Checks to see whether the input file is an excluded file
+        //! Checks to see whether the input file is an excluded file, assumes input is absolute path.
         bool IsFileExcluded(QString fileName) const;
+        //! If you already have a relative path, this is a cheaper function to call:
+        bool IsFileExcludedRelPath(QString relPath) const;
 
         //! Given a file name, return a container that contains all matching recognizers
         //!
@@ -295,7 +297,8 @@ namespace AssetProcessor
 
         void PopulatePlatformsForScanFolder(AZStd::vector<AssetBuilderSDK::PlatformInfo>& platformsList, QStringList includeTagsList = QStringList(), QStringList excludeTagsList = QStringList());
 
-        void CacheIntermediateAssetsScanFolderId();
+        // uses const + mutability since its a cache.
+        void CacheIntermediateAssetsScanFolderId() const; 
         AZStd::optional<AZ::s64> GetIntermediateAssetsScanFolderId() const;
 
     protected:
@@ -323,7 +326,7 @@ namespace AssetProcessor
         QList<QPair<QString, QString> > m_metaDataFileTypes;
         QSet<QString> m_metaDataRealFiles;
         AZStd::vector<AzFramework::GemInfo> m_gemInfoList;
-        AZ::s64 m_intermediateAssetScanFolderId = -1; // Cached ID for intermediate scanfolder, for quick lookups
+        mutable AZ::s64 m_intermediateAssetScanFolderId = -1; // Cached ID for intermediate scanfolder, for quick lookups
 
         int m_minJobs = 1;
         int m_maxJobs = 3;

Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp

@@ -822,25 +822,32 @@ namespace AssetUtilities
     QString NormalizeFilePath(const QString& filePath)
     {
         // do NOT convert to absolute paths here, we just want to manipulate the string itself.
+        QString returnString = filePath;
 
-        // note that according to the Qt Documentation, in QDir::toNativeSeparators,
-        // "The returned string may be the same as the argument on some operating systems, for example on Unix.".
-        // in other words, what we need here is a custom normalization - we want always the same
-        // direction of slashes on all platforms.s
+        // QDir::cleanPath only replaces backslashes with forward slashes in the input string if the OS
+        // it is currently natively running on uses backslashes as its native path separator.
+        // see https://github.com/qt/qtbase/blob/40143c189b7c1bf3c2058b77d00ea5c4e3be8b28/src/corelib/io/qdir.cpp#L2357
+        // This assumption is incorrect in this application - it can receive file paths from data files created on
+        // backslash operating systems even if its a non-backslash operating system.
 
-        QString returnString = filePath;
-        returnString.replace(QChar('\\'), QChar('/'));
+        // we can skip this step in the cases where cleanPath will do it for us:
+        if (QDir::separator() == QLatin1Char('/'))
+        {
+            returnString.replace(QLatin1Char('\\'), QLatin1Char('/'));
+        }
+
+        // cleanPath to remove/resolve .. and . and any extra slashes, and remove any trailing slashes.
         returnString = QDir::cleanPath(returnString);
 
 #if defined(AZ_PLATFORM_WINDOWS)
         // windows has an additional idiosyncrasy - it returns upper and lower case drive letters
         // from various APIs differently.  we will settle on upper case as the standard.
-        if ((returnString.length() > 1) && (returnString[1] == ':'))
+        if ((returnString.length() > 1) && (returnString.at(1) == ':'))
         {
-            returnString[0] = returnString[0].toUpper();
+            QCharRef firstChar = returnString[0]; // QCharRef allows you to modify the string in place.
+            firstChar = firstChar.toUpper();
         }
 #endif
-
         return returnString;
     }
 

Gems/Terrain/Registry/Platform/Mac/AssetProcessorPlatformConfig.setreg

@@ -5,10 +5,10 @@
                 // The terrain shader doesn't work on mac due to unbounded arrays, so disable problematic materials and material types
                 // in the terrain gem to prevent dependencies from failing.
                 "Exclude Terrain DefaultPbrTerrain.material": {
-                    "pattern": "^Materials/Terrain/DefaultPbrTerrain.material"
+                    "glob": "Materials/Terrain/DefaultPbrTerrain.material"
                 },
                 "Exclude Terrain PbrTerrain.materialtype": {
-                    "pattern": "^Materials/Terrain/PbrTerrain.materialtype"
+                    "glob": "Materials/Terrain/PbrTerrain.materialtype"
                 }
             }
         }

Registry/AssetProcessorPlatformConfig.setreg

@@ -155,69 +155,48 @@
                     "order": 40000
                 },
 
-                // Excludes files that match the pattern or glob 
+                // Excludes files that match the pattern or glob.
+                // the input string will be the relative path from the scan folder the file was found in.
+                // patterns are case sensitive regular expressions, while globs are simple wildcard matches (non-case-sensitive)
                 // if you use a pattern, remember to escape your backslashes (\\)
-                // The patterns are checked against a path relative to the entry's
-                // root scan folder.
                 "Exclude _LevelBackups": {
-                    "pattern": "(^|.+/)Levels/.*/_savebackup(/.*)?$"
+                    "glob": "*/_savebackup/*"
                 },
                 "Exclude _LevelAutoBackups": {
-                    "pattern": "(^|.+/)Levels/.*/_autobackup(/.*)?$"
-                },
-                "Exclude HoldFiles": {
-                    "pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
+                    "glob": "*/_autobackup/*"
                 },
                 // note that $ has meaning to regex, so we escape it.
                 "Exclude TempFiles": {
                     "pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
                 },
-                "Exclude TmpAnimationCompression": {
-                    "pattern": "(^|.+/)Editor/Tmp/AnimationCompression(/.*)?$"
-                },
                 "Exclude EventLog": {
-                    "pattern": "(^|.+/)Editor/.*eventlog\\\\.xml"
+                    "glob": "*eventlog.xml"
                 },
                 "Exclude GameGemsCode": {
-                    "pattern": "(^|.+/)Gem/Code(/.*)?$"
+                    "glob": "Gem/Code/*"
                 },
                 "Exclude GameGemsResources": {
-                    "pattern": "(^|.+/)Gem/Resources(/.*)?$"
+                    "glob": "Gem/Resources/*"
                 },
                 "Exclude Private Certs": {
-                    "pattern": "(^|.+/)DynamicContent/Certificates/Private(/.*)?$"
+                    "glob": "*DynamicContent/Certificates/Private*"
                 },
                 "Exclude CMakeLists": {
-                    "pattern": "(^|.+/)CMakeLists\\\\.txt"
+                    "glob": "*CMakeLists.txt"
                 },
                 "Exclude CMakeFiles": {
-                    "pattern": "(^|.+/).+\\\\.cmake"
+                    "glob": "*.cmake"
                 },
                 "Exclude User": {
-                    "pattern": "^[Uu]ser(/.*)?$"
+                    "glob": "user/*"
                 },
                 "Exclude Build": {
-                    "pattern": "^[Bb]uild(/.*)?$"
+                    "glob": "build/*"
                 },
                 "Exclude Install": {
-                    "pattern": "^[Ii]nstall(/.*)?$"
-                },
-                "Exclude UserSettings": {
-                    "pattern": "(^|[^/]+/)UserSettings\\\\.xml"
+                    "glob": "install/*"
                 },
 
-                // ------------------------------------------------------------------------------
-                // Large Worlds Test
-                // ------------------------------------------------------------------------------
-                "Exclude Work In Progress Folders": {
-                    "pattern": "(^|[^/]+/)WIP(/.*)?"
-                },
-                "Exclude Content Source Folders": {
-                    "pattern": "(^|[^/]+/)CONTENT_SOURCE(/.*)?"
-                },
-                "Exclude Art Source Folders": {
-                    "pattern": "(^|[^/]+/)ArtSource(/.*)?"
-                },
                 //------------------------------------------------------------------------------
                 // Copying Files Automatically Into the Cache
                 //------------------------------------------------------------------------------
@@ -229,6 +208,8 @@
                 // and is the recommended way to specify that a certain kind of file (such as '*.myextension') becomes registered as the
                 // actual UUID of that type in the engine itself.
 
+                // globs ("*.txt") are usually faster to match (CPU wise) and easier for humans to understand than patterns.
+
                 // Use a regex for matching files, same params for all platforms
                 // "RC TGAs": {
                 //  "pattern": ".+\\\\.tga$",