Pass relative path IsFileExcluded so that only paths relative to the root scan folder are matched against the exclude filters (#4504)

* Pass relative path IsFileExcluded so that only paths relative to the root scan folder are matched against the exclude filters.

Signed-off-by: amzn-sj <[email protected]>

* Revert previous change. Remove the exclude filter for the Install directory.

Signed-off-by: amzn-sj <[email protected]>

* Pass in relative path to the exclude filter as before. Fix the AssetScanner tests.

Signed-off-by: amzn-sj <[email protected]>

* Prepend a ./ to the relative path in order to match the exclude patterns

Signed-off-by: amzn-sj <[email protected]>

* Remove hack to prepend ./. Update the exclude patterns so that the hack is no longer required.

Signed-off-by: amzn-sj <[email protected]>

* Add missing ? and remove whitespace

Signed-off-by: amzn-sj <[email protected]>

* 1. IsFileExcluded() now converts the input path to a path that's relative to its corresponding scan folder.
2. Update regex patterns in gems and AutomatedTesting as well.
3. Remove unnecessary escaping for '/'.

Signed-off-by: amzn-sj <[email protected]>

* Use ConvertToRelativePath() function to compute path relative to a scan folder.

Signed-off-by: amzn-sj <[email protected]>

* More fixes to regex patterns

Signed-off-by: amzn-sj <[email protected]>

* Remove test case which tests a hypothetical scenario that cannot occur. Fix another test case by adding scan folder.

Signed-off-by: amzn-sj <[email protected]>

* Remove assert that's not needed since it's a valid scenario

Signed-off-by: amzn-sj <[email protected]>

AutomatedTesting/Gem/AssetProcessorGemConfig.setreg

@@ -3,19 +3,19 @@
         "AssetProcessor": {
             "Settings": {
                 "Exclude PythonTest Benchmark Settings Assets": {
-                    "pattern": ".*\\\\/PythonTests\\\\/.*benchmarksettings"
+                    "pattern": "(^|.+/)PythonTests/.*benchmarksettings"
                 },
                 "Exclude fbx_tests": {
-                    "pattern": ".*\\\\/fbx_tests\\\\/assets\\\\/.*"
+                    "pattern": "(^|.+/)fbx_tests/assets(/.+)$"
                 },
                 "Exclude wwise_bank_dependency_tests": {
-                    "pattern": ".*\\\\/wwise_bank_dependency_tests\\\\/assets\\\\/.*"
+                    "pattern": "(^|.+/)wwise_bank_dependency_tests/assets(/.+)$"
                 },
                 "Exclude AssetProcessorTestAssets": {
-                    "pattern": ".*\\\\/asset_processor_tests\\\\/assets\\\\/.*"
+                    "pattern": "(^|.+/)asset_processor_tests/assets(/.+)$"
                 },
                 "Exclude Restricted AssetProcessorTestAssets": {
-                    "pattern": ".*\\\\/asset_processor_tests\\\\/restricted\\\\/.*"
+                    "pattern": "(^|.+/)asset_processor_tests/restricted(/.+)$"
                 }
             }
         }

Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp

@@ -281,7 +281,7 @@ namespace AssetProcessor
 
             ExcludeAssetRecognizer excludeRecogniser;
             excludeRecogniser.m_name = "backup";
-            excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/savebackup\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
+            excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|.+/)savebackup/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
             config.AddExcludeRecognizer(excludeRecogniser);
         }
 

Code/Tools/AssetProcessor/native/tests/assetscanner/AssetScannerTests.cpp

@@ -123,7 +123,7 @@ namespace AssetProcessor
         ExcludeAssetRecognizer excludeRecogniser;
         excludeRecogniser.m_name = "backup";
         // we are excluding all the files in the folder but not the folder itself
-        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/subfolder2\\/aaa\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
+        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|[^/]+/)aaa/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
         m_platformConfig.get()->AddExcludeRecognizer(excludeRecogniser);
         m_assetScanner.get()->StartScan();
 
@@ -144,7 +144,7 @@ namespace AssetProcessor
         ExcludeAssetRecognizer excludeRecogniser;
         excludeRecogniser.m_name = "backup";
         // we are excluding the complete folder here
-        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/subfolder2\\/aaa", AssetBuilderSDK::AssetBuilderPattern::Regex);
+        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|[^/]+/)aaa", AssetBuilderSDK::AssetBuilderPattern::Regex);
         m_platformConfig.get()->AddExcludeRecognizer(excludeRecogniser);
         m_assetScanner.get()->StartScan();
 

Code/Tools/AssetProcessor/native/tests/platformconfiguration/platformconfigurationtests.cpp

@@ -405,6 +405,8 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_RegularExcludes)
     auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
     ASSERT_TRUE(configRoot);
     UnitTestPlatformConfiguration config;
+    
+    config.AddScanFolder(ScanFolderInfo("blahblah", "Blah ScanFolder", "sf2", true, true), true);
     m_absorber.Clear();
     ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
     ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);

Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp

@@ -347,7 +347,7 @@ namespace AssetProcessor
 
         ExcludeAssetRecognizer excludeRecogniser;
         excludeRecogniser.m_name = "backup";
-        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/savebackup\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
+        excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|.+/)savebackup/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
         config.AddExcludeRecognizer(excludeRecogniser);
 
         AssetProcessorManager_Test apm(&config);  // note, this will 'push' the scan folders in to the db.
@@ -1791,13 +1791,8 @@ namespace AssetProcessor
         UNIT_TEST_EXPECT_TRUE((newfingerprintForPCAfterVersionChange != fingerprintForPC) || (newfingerprintForPCAfterVersionChange != newfingerprintForPC));//Fingerprints should be different
         UNIT_TEST_EXPECT_TRUE((newfingerprintForANDROIDAfterVersionChange != fingerprintForANDROID) || (newfingerprintForANDROIDAfterVersionChange != newfingerprintForANDROID));//Fingerprints should be different
 
-        //------Test for Files which are excluded
         processResults.clear();
-        absolutePath = AssetUtilities::NormalizeFilePath(tempPath.absoluteFilePath("subfolder3/savebackup/test.txt"));
-        QMetaObject::invokeMethod(&apm, "AssessModifiedFile", Qt::QueuedConnection, Q_ARG(QString, absolutePath));
-        UNIT_TEST_EXPECT_FALSE(BlockUntil(idling, 3000)); //Processing a file that will be excluded should not cause assetprocessor manager to emit the onBecameIdle signal because its state should not change
-        UNIT_TEST_EXPECT_TRUE(processResults.size() == 0);
-
+        
         // ------------- Test querying asset status -------------------
         {
             absolutePath = tempPath.absoluteFilePath("subfolder2/folder/ship.tiff");

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp

@@ -1633,13 +1633,18 @@ namespace AssetProcessor
 
     bool AssetProcessor::PlatformConfiguration::IsFileExcluded(QString fileName) const
     {
-        for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
+        QString relPath, scanFolderName;
+        if (ConvertToRelativePath(fileName, relPath, scanFolderName))
         {
-            if (excludeRecognizer.m_patternMatcher.MatchesPath(fileName.toUtf8().constData()))
+            for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
             {
-                return true;
+                if (excludeRecognizer.m_patternMatcher.MatchesPath(relPath.toUtf8().constData()))
+                {
+                    return true;
+                }
             }
         }
+
         return false;
     }
 

Code/Tools/AssetProcessor/testdata/config_regular/AssetProcessorPlatformConfig.setreg

@@ -50,10 +50,10 @@
                     "order": 6000
                 },
                 "Exclude HoldFiles": {
-                    "pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
+                    "pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
                 },
                 "Exclude TempFiles": {
-                    "pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
+                    "pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
                 },
                 "RC i_caf": {
                     "glob": "*.i_caf",

Code/Tools/AssetProcessor/testdata/config_regular_platform_scanfolder/AssetProcessorPlatformConfig.setreg

@@ -74,10 +74,10 @@
                     "include": "test"
                 },
                 "Exclude HoldFiles": {
-                    "pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
+                    "pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
                 },
                 "Exclude TempFiles": {
-                    "pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
+                    "pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
                 },
                 "RC i_caf": {
                     "glob": "*.i_caf",

Gems/AtomContent/Sponza/Registry/AssetProcessorPlatformConfig.setreg

@@ -6,7 +6,7 @@
                 // Sample Gems, Block source folders
                 // ------------------------------------------------------------------------------
                 "Exclude Work In Progress Folders": {
-                    "pattern": ".*\\\\/.[Ss]rc\\\\/.*"
+                    "pattern": "(^|.+/).[Ss]rc(/.*)?$"
                 }
             }
         }

Gems/AudioEngineWwise/AssetProcessorGemConfig.setreg

@@ -3,10 +3,10 @@
         "AssetProcessor": {
             "Settings": {
                 "Exclude AudioProject": {
-                    "pattern": ".*\\\\/Sounds\\\\/.+_project.*"
+                    "pattern": "(^|.+/)Sounds/.+_project.*"
                 },
                 "Exclude AudioTemp": {
-                    "pattern": ".*\\\\/Sounds\\\\/.+\\\\.(txt|xml|dat)"
+                    "pattern": "(^|.+/)Sounds/.+\\\\.(txt|xml|dat)"
                 },
                 "RC audio": {
                     "pattern": ".*\\\\.(wav|pcm)",

Registry/AssetProcessorPlatformConfig.setreg

@@ -137,64 +137,66 @@
 
                 // Excludes files that match the pattern or glob 
                 // if you use a pattern, remember to escape your backslashes (\\)
+                // The patterns are checked against a path relative to the entry's
+                // root scan folder.
                 "Exclude _LevelBackups": {
-                    "pattern": ".*\\\\/Levels\\\\/.*\\\\/_savebackup\\\\/.*"
+                    "pattern": "(^|.+/)Levels/.*/_savebackup(/.*)?$"
                 },
                 "Exclude _LevelAutoBackups": {
-                    "pattern": ".*\\\\/Levels\\\\/.*\\\\/_autobackup\\\\/.*"
+                    "pattern": "(^|.+/)Levels/.*/_autobackup(/.*)?$"
                 },
                 "Exclude HoldFiles": {
-                    "pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
+                    "pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
                 },
                 // note that $ has meaning to regex, so we escape it.
                 "Exclude TempFiles": {
-                    "pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
+                    "pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
                 },
                 "Exclude TmpAnimationCompression": {
-                    "pattern": ".*\\\\/Editor\\\\/Tmp\\\\/AnimationCompression\\\\/.*"
+                    "pattern": "(^|.+/)Editor/Tmp/AnimationCompression(/.*)?$"
                 },
                 "Exclude EventLog": {
-                    "pattern": ".*\\\\/Editor\\\\/.*eventlog\\\\.xml"
+                    "pattern": "(^|.+/)Editor/.*eventlog\\\\.xml"
                 },
                 "Exclude GameGemsCode": {
-                    "pattern": ".*\\\\/Gem\\\\/Code\\\\/.*"
+                    "pattern": "(^|.+/)Gem/Code(/.*)?$"
                 },
                 "Exclude GameGemsResources": {
-                    "pattern": ".*\\\\/Gem\\\\/Resources\\\\/.*"
+                    "pattern": "(^|.+/)Gem/Resources(/.*)?$"
                 },
                 "Exclude Private Certs": {
-                    "pattern": ".*\\DynamicContent\\\\/Certificates\\\\/Private\\\\/.*"
+                    "pattern": "(^|.+/)DynamicContent/Certificates/Private(/.*)?$"
                 },
                 "Exclude CMakeLists": {
-                    "pattern": ".*\\\\/CMakeLists.txt"
+                    "pattern": "(^|.+/)CMakeLists\\\\.txt"
                 },
                 "Exclude CMakeFiles": {
-                    "pattern": ".*\\\\/.*\\\\.cmake"
+                    "pattern": "(^|.+/).+\\\\.cmake"
                 },
                 "Exclude User": {
-                    "pattern": ".*/[Uu]ser/.*"
+                    "pattern": "^[Uu]ser(/.*)?$"
                 },
                 "Exclude Build": {
-                    "pattern": ".*/[Bb]uild/.*"
+                    "pattern": "^[Bb]uild(/.*)?$"
                 },
                 "Exclude Install": {
-                    "pattern": ".*/[Ii]nstall/.*"
+                    "pattern": "^[Ii]nstall(/.*)?$"
                 },
                 "Exclude UserSettings": {
-                    "pattern": ".*/UserSettings.xml"
+                    "pattern": "(^|[^/]+/)UserSettings\\\\.xml"
                 },
 
                 // ------------------------------------------------------------------------------
                 // Large Worlds Test
                 // ------------------------------------------------------------------------------
                 "Exclude Work In Progress Folders": {
-                    "pattern": ".*\\\\/WIP\\\\/.*"
+                    "pattern": "(^|[^/]+/)WIP(/.*)?"
                 },
                 "Exclude Content Source Folders": {
-                    "pattern": ".*\\\\/CONTENT_SOURCE\\\\/.*"
+                    "pattern": "(^|[^/]+/)CONTENT_SOURCE(/.*)?"
                 },
                 "Exclude Art Source Folders": {
-                    "pattern": ".*\\\\/ArtSource\\\\/.*"
+                    "pattern": "(^|[^/]+/)ArtSource(/.*)?"
                 },
                 //------------------------------------------------------------------------------
                 // Copying Files Automatically Into the Cache