Allow adjusting scan folder priority for project-relative Gems (#14028)

* Add functionality to configure priority of gems

- For scan folders of Gems inside the project, this allows you more
  flexibility to configure the priorities of the scan folders, with
  respect to the main project scan folder.
- "GemScanFolderPriorityStart": the starting value of all gems priority
- "ProjectGemsRelativeScanFolderPriority": controls whether
  project-relative gems will be prioritized higher, lower, or "none"
  against the project scan folder.

Signed-off-by: amzn-phist <[email protected]>

* Renamed the settings registry keys

- Also updated comments

Signed-off-by: amzn-phist <[email protected]>

* Simplified comments

Signed-off-by: amzn-phist <[email protected]>

* Removes old gem order global variable

- Fixes linux build because it's unused.

Signed-off-by: amzn-phist <[email protected]>

* Addressing PR feedback 1

- Fix the FindScanFolder iterator & return.
- Change an index-based find function to use FindScanFolder.
- Add string constants.
- Check for gem paths that are relative to the project path (simplify).

Signed-off-by: amzn-phist <[email protected]>

* Minor typo

Signed-off-by: amzn-phist <[email protected]>

Signed-off-by: amzn-phist <[email protected]>

Code/Framework/AzCore/AzCore/Settings/SettingsRegistry.h

@@ -247,7 +247,7 @@ namespace AZ
         //! Register a post-merge hahndler with the PostMergeEvent.
         //! The handler will be called after a file is merged.
         //! @param handler The handler to register with the PostmergeEVent.
-        virtual void RegisterPostMergeEvent(PostMergeEventHandler& hanlder) = 0;
+        virtual void RegisterPostMergeEvent(PostMergeEventHandler& handler) = 0;
 
         //! Gets the boolean value at the provided path.
         //! @param result The target to write the result to.
@@ -278,7 +278,7 @@ namespace AZ
         //! @param resultTypeId The type id of the target that's being written to.
         //! @param path The path to the value.
         //! @return Whether or not the value was stored. An invalid path will return false;
-        virtual bool GetObject(void* result, AZ::Uuid resultTypeID, AZStd::string_view path) const = 0;
+        virtual bool GetObject(void* result, AZ::Uuid resultTypeId, AZStd::string_view path) const = 0;
         //! Gets the json object value at the provided path serialized to the target struct/class. Classes retrieved
         //! through this call needs to be registered with the Serialize Context.
         //! @param result The target to write the result to.
@@ -322,13 +322,13 @@ namespace AZ
         //! @param value The new value to store.
         //! @param valueTypeId The type id of the target that's being stored.
         //! @return Whether or not the value was stored. An invalid path will return false;
-        virtual bool SetObject(AZStd::string_view path, const void* value, AZ::Uuid valueTypeID) = 0;
-        template<typename T>
+        virtual bool SetObject(AZStd::string_view path, const void* value, AZ::Uuid valueTypeId) = 0;
         //! Sets the value at the provided path to the serialized version of the provided struct/class.
         //! Classes used for this call need to be registered with the Serialize Context.
         //! @param path The path to the value.
         //! @param value The new value to store.
         //! @return Whether or not the value was stored. An invalid path will return false;
+        template<typename T>
         bool SetObject(AZStd::string_view path, const T& value) { return SetObject(path, &value, azrtti_typeid(value)); }
 
         //! Remove the value at the provided path 

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp

@@ -21,11 +21,6 @@
 
 #include <AzCore/Serialization/SerializeContext.h>
 
-namespace
-{
-    // the starting order in the file for gems.
-    const int g_gemStartingOrder = 100;
-}
 
 namespace AssetProcessor
 {
@@ -624,6 +619,9 @@ namespace AssetProcessor
 
     const char AssetConfigPlatformDir[] = "AssetProcessorConfig/";
     const char AssetProcessorPlatformConfigFileName[] = "AssetProcessorPlatformConfig.ini";
+    constexpr const char* ProjectScanFolderKey = "Project/Assets";
+    constexpr const char* GemStartingPriorityOrderKey = "/GemScanFolderStartingPriorityOrder";
+    constexpr const char* ProjectRelativeGemPriorityKey = "/ProjectRelativeGemsScanFolderPriority";
 
     PlatformConfiguration::PlatformConfiguration(QObject* pParent)
         : QObject(pParent)
@@ -1326,6 +1324,19 @@ namespace AssetProcessor
         }
     }
 
+    int PlatformConfiguration::GetProjectScanFolderOrder() const
+    {
+        auto mainProjectScanFolder = FindScanFolder([](const AssetProcessor::ScanFolderInfo& scanFolderInfo) -> bool
+            {
+                return scanFolderInfo.GetPortableKey() == ProjectScanFolderKey;
+            });
+        if (mainProjectScanFolder)
+        {
+            return mainProjectScanFolder->GetOrder();
+        }
+        return 0;
+    }
+
     bool PlatformConfiguration::MergeConfigFileToSettingsRegistry(AZ::SettingsRegistryInterface& settingsRegistry, const AZ::IO::PathView& configFile)
     {
         // If the config file is a settings registry file use the SettingsRegistryInterface MergeSettingsFile function
@@ -1446,17 +1457,19 @@ namespace AssetProcessor
         return m_scanFolders[index];
     }
 
+    const AssetProcessor::ScanFolderInfo* PlatformConfiguration::FindScanFolder(
+        AZStd::function<bool(const AssetProcessor::ScanFolderInfo&)> predicate) const
+    {
+        auto resultIt = AZStd::ranges::find_if(m_scanFolders, predicate);
+        return resultIt != m_scanFolders.end() ? &(*resultIt) : nullptr;
+    }
+
     const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderById(AZ::s64 id) const
     {
-        auto* result = AZStd::find_if(
-            m_scanFolders.begin(),
-            m_scanFolders.end(),
-            [id](const ScanFolderInfo& scanFolder)
+        return FindScanFolder([id](const ScanFolderInfo& scanFolder)
             {
                 return scanFolder.ScanFolderID() == id;
             });
-
-        return result != m_scanFolders.end() ? result : nullptr;
     }
 
     void PlatformConfiguration::AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting)
@@ -1595,7 +1608,7 @@ namespace AssetProcessor
         auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
 
         // Only compute the intermediate assets folder path if we are going to search for and skip it.
-       
+
         if (skipIntermediateScanFolder)
         {
             if (m_intermediateAssetScanFolderId == -1)
@@ -1603,13 +1616,13 @@ namespace AssetProcessor
                 CacheIntermediateAssetsScanFolderId();
             }
         }
-        
+
         QString absolutePath; // avoid allocating memory repeatedly here by reusing absolutePath each scan folder.
         absolutePath.reserve(AZ_MAX_PATH_LEN);
 
         QFileInfo details(relativeName); // note that this does not actually hit the actual storage medium until you query something
         bool isAbsolute = details.isAbsolute(); // note that this looks at the file name string only, it does not hit storage.
-        
+
         for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
         {
             const AssetProcessor::ScanFolderInfo& scanFolderInfo = m_scanFolders[pathIdx];
@@ -1807,14 +1820,10 @@ namespace AssetProcessor
     //! Given a scan folder path, get its complete info
     const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderByPath(const QString& scanFolderPath) const
     {
-        for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
-        {
-            if (m_scanFolders[pathIdx].ScanPath() == scanFolderPath)
+        return FindScanFolder([&scanFolderPath](const AssetProcessor::ScanFolderInfo& scanFolder)
             {
-                return &m_scanFolders[pathIdx];
-            }
-        }
-        return nullptr;
+                return scanFolder.ScanPath() == scanFolderPath;
+            });
     }
 
     int PlatformConfiguration::GetMinJobs() const
@@ -1863,7 +1872,48 @@ namespace AssetProcessor
 
     void PlatformConfiguration::AddGemScanFolders(const AZStd::vector<AzFramework::GemInfo>& gemInfoList)
     {
-        int gemOrder = g_gemStartingOrder;
+        // If the gem is project-relative, make adjustments to its priority order based on registry settings:
+        // /Amazon/AssetProcessor/Settings/GemScanFolderStartingPriorityOrder
+        // /Amazon/AssetProcessor/Settings/ProjectRelativeGemsScanFolderPriority
+        // See <o3de-root>/Registry/AssetProcessorPlatformConfig.setreg for more information.
+
+        AZ::s64 gemStartingOrder = 100;
+        AZStd::string projectGemPrioritySetting{};
+        const AZ::IO::FixedMaxPath projectPath = AZ::Utils::GetProjectPath();
+        int pathCount = 0;
+
+        const int projectScanOrder = GetProjectScanFolderOrder();
+
+        if (auto const settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
+        {
+            settingsRegistry->Get(gemStartingOrder,
+                AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorSettingsKey) + GemStartingPriorityOrderKey);
+
+            settingsRegistry->Get(projectGemPrioritySetting,
+                AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorSettingsKey) + ProjectRelativeGemPriorityKey);
+            AZStd::to_lower(projectGemPrioritySetting.begin(), projectGemPrioritySetting.end());
+        }
+
+        auto GetGemFolderOrder = [&](bool isProjectRelativeGem) -> int
+        {
+            ++pathCount;
+            int currentGemOrder = aznumeric_cast<int>(gemStartingOrder) + pathCount;
+            if (isProjectRelativeGem)
+            {
+                if (projectGemPrioritySetting == "higher")
+                {
+                    currentGemOrder = projectScanOrder - pathCount;
+                }
+                else if (projectGemPrioritySetting == "lower")
+                {
+                    currentGemOrder = projectScanOrder + pathCount;
+                }
+            }
+            return currentGemOrder;
+        };
+
+        int gemOrder = aznumeric_cast<int>(gemStartingOrder);
+
         AZStd::vector<AssetBuilderSDK::PlatformInfo> platforms;
         PopulatePlatformsForScanFolder(platforms);
 
@@ -1873,6 +1923,9 @@ namespace AssetProcessor
             {
                 const AZ::IO::Path& absoluteSourcePath = gemElement.m_absoluteSourcePaths[sourcePathIndex];
                 QString gemAbsolutePath = QString::fromUtf8(absoluteSourcePath.c_str(), aznumeric_cast<int>(absoluteSourcePath.Native().size())); // this is an absolute path!
+
+                const bool isProjectGem = absoluteSourcePath.IsRelativeTo(projectPath);
+
                 // Append the index of the source path array element to make a unique portable key is created for each path of a gem
                 AZ::Uuid gemNameUuid = AZ::Uuid::CreateName((gemElement.m_gemName + AZStd::to_string(sourcePathIndex)).c_str());
                 QString gemNameAsUuid(gemNameUuid.ToFixedString().c_str());
@@ -1894,7 +1947,7 @@ namespace AssetProcessor
                 QString portableKey = QString("gemassets-%1").arg(gemNameAsUuid);
                 bool isRoot = false;
                 bool isRecursive = true;
-                gemOrder++;
+                gemOrder = GetGemFolderOrder(isProjectGem);
 
                 AZ_TracePrintf(AssetProcessor::DebugChannel, "Adding GEM assets folder for monitoring / scanning: %s.\n", gemFolder.toUtf8().data());
                 AddScanFolder(ScanFolderInfo(
@@ -1914,7 +1967,7 @@ namespace AssetProcessor
 
                 assetBrowserDisplayName = AzFramework::GemInfo::GetGemRegistryFolder();
                 portableKey = QString("gemregistry-%1").arg(gemNameAsUuid);
-                gemOrder++;
+                gemOrder = GetGemFolderOrder(isProjectGem);
 
                 AZ_TracePrintf(AssetProcessor::DebugChannel, "Adding GEM registry folder for monitoring / scanning: %s.\n", gemFolder.toUtf8().data());
                 AddScanFolder(ScanFolderInfo(

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h

@@ -196,7 +196,8 @@ namespace AssetProcessor
 
         //! Retrieve the scan folder at a given index.
         const AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index) const;
-
+        //! Retrieve the scan folder found by a boolean predicate function, when the predicate returns true, the current scan folder info is returned.
+        const AssetProcessor::ScanFolderInfo* FindScanFolder(AZStd::function<bool(const AssetProcessor::ScanFolderInfo&)> predicate) const;
         const AssetProcessor::ScanFolderInfo* GetScanFolderById(AZ::s64 id) const override;
 
         //!  Manually add a scan folder.  Also used for testing.
@@ -262,11 +263,11 @@ namespace AssetProcessor
         //! c:/dev/engine/models/box01.mdl
         //! ----> [models/box01.mdl] found under[c:/dev/engine]
         //! note that this does return a database source path by default
-        bool ConvertToRelativePath(QString fullFileName, QString& databaseSourceName, QString& scanFolderName) const;
+        bool ConvertToRelativePath(QString fullFileName, QString& databaseSourceName, QString& scanFolderName) const override;
         static bool ConvertToRelativePath(const QString& fullFileName, const ScanFolderInfo* scanFolderInfo, QString& databaseSourceName);
 
         //! given a full file name (assumed already fed through the normalization funciton), return the first matching scan folder
-        const AssetProcessor::ScanFolderInfo* GetScanFolderForFile(const QString& fullFileName) const;
+        const AssetProcessor::ScanFolderInfo* GetScanFolderForFile(const QString& fullFileName) const override;
 
         //! Given a scan folder path, get its complete info
         const AssetProcessor::ScanFolderInfo* GetScanFolderByPath(const QString& scanFolderPath) const;
@@ -317,6 +318,8 @@ namespace AssetProcessor
         bool ReadRecognizersFromSettingsRegistry(const QString& assetRoot, bool skipScanFolders = false, QStringList scanFolderPatterns = QStringList() );
         void ReadMetaDataFromSettingsRegistry();
 
+        int GetProjectScanFolderOrder() const;
+
     private:
         AZStd::vector<AssetBuilderSDK::PlatformInfo> m_enabledPlatforms;
         RecognizerContainer m_assetRecognizers;

Registry/AssetProcessorPlatformConfig.setreg

@@ -1,5 +1,5 @@
 {
-    // ---- Enable/Disable platforms for the entire project. AssetProcessor will automatically add the current platform by default. 
+    // ---- Enable/Disable platforms for the entire project. AssetProcessor will automatically add the current platform by default.
 
     // PLATFORM DEFINITIONS
     // [Platform (unique identifier)]
@@ -53,7 +53,7 @@
                 "Platform mac": {
                     "tags": "tools,renderer,metal,null"
                 },
-                // this is an example of a headless platform that has no renderer.   
+                // this is an example of a headless platform that has no renderer.
                 // To use this you would still need to make sure 'assetplatform' in your startup params in your main() chooses this 'server' platform as your server 'assets' flavor
                 "Platform server": {
                     "tags": "server,dx12,vulkan"
@@ -89,7 +89,7 @@
                 // however if your metafile REPLACES the extension (for example, if you have the file blah.i_caf and its metafile is blah.exportsettings)
                 // then you specify the original extension here to narrow the scope.
                 // If a relative path to a specific file is provided instead of an extension, a change to the file will change all files
-                // with the associated extension (e.g. Animations/SkeletonList.xml=i_caf will cause all i_caf files to recompile when 
+                // with the associated extension (e.g. Animations/SkeletonList.xml=i_caf will cause all i_caf files to recompile when
                 // Animations/SkeletonList.xml within the current game project changes)
 
                 "MetaDataTypes": {
@@ -104,21 +104,21 @@
                 },
 
                 // ---- add any folders to scan here.  The priority order is the order they appear here
-                // available macros are 
+                // available macros are
                 // @ROOT@ - the location of asset root
-                // @PROJECTROOT@ - the location of the project root, for example 'Q:\MyProjects\RPGSample' 
+                // @PROJECTROOT@ - the location of the project root, for example 'Q:\MyProjects\RPGSample'
                 // note that they are sorted by their 'order' value, and the lower the order the more important an asset is
                 // lower order numbers override higher ones.
                 // If specified, output will be prepended to every path found in that recognizer's watch folder.
                 // Note that you can also make the scan folder platform specific by using the keywords include and exclude.
                 // Both include and exclude can contain either platform tags, platform identifiers or both.
                 // if no include is specified, all currently enabled platforms are included by default.
-                // If includes ARE specified, it will be filtered down by the list of currently enabled platforms. 
+                // If includes ARE specified, it will be filtered down by the list of currently enabled platforms.
                 // "ScanFolder (unique identifier)": {
                 //  "include": "(comma seperated platform tags or identifiers)",
                 //  "exclude": "(comma seperated platform tags or identifiers)"
                 // }
-                // For example if you want to include a scan folder only for platforms that have the platform tags tools and renderer 
+                // For example if you want to include a scan folder only for platforms that have the platform tags tools and renderer
                 // but omit it for platform mac, you will have a scanfolder rule like
                 // "ScanFolder (unique identifier)": {
                 // "watch": "@ROOT@/foo",
@@ -132,7 +132,6 @@
                     "recursive": 1,
                     "order": 0
                 },
-                // gems will be auto-added from 100 onwards
                 "ScanFolder Root": {
                     "watch": "@ROOT@",
                     "recursive": 0,
@@ -155,6 +154,17 @@
                     "order": 40000
                 },
 
+                // Configurable starting priority for all Gem scan folders to use.
+                // Each Gem scan folder added will increment this.
+                // NOTE: Each successive gem priority will be lower than the last.
+                "GemScanFolderStartingPriorityOrder": 100,
+
+                // Control how project-relative Gem scan folders are prioritized against the project's main scan folder (key: "Project/Assets", alias: @PROJECTROOT@).
+                //  "none" - any project Gem scan folder priority will be incremented from the "GemScanFolderStartingPriorityOrder" value (default behavior).
+                //  "lower" - each project Gem scan folder will be set to lower priority (higher numeric value) than the project scan folder.
+                //  "higher" - each project Gem scan folder will be set to higher priority (lower numeric value) than the project scan folder.
+                "ProjectRelativeGemsScanFolderPriority": "none",
+
                 // Excludes files that match the pattern or glob.
                 // the input string will be the relative path from the scan folder the file was found in.
                 // patterns are case sensitive regular expressions, while globs are simple wildcard matches (non-case-sensitive)
@@ -239,14 +249,14 @@
                 // so ensure start your patterns with .* or as appropriate.
                 // Also, any rules which match will apply - so if you have two rules which both apply to PNG files for example
                 // but you only want one, you might want to use exclusion patterns:
-                 
+
                 //Example: copy everything EXCEPT the ones in the libs/ui
                 // "RC png-normal": {
                 //  "pattern": "(?!.*libs\\\\/ui\\\\/).*\\.png",
                 //  "params": "copy"
                 //}
 
-                //Example:  Process everything in the libs/ui folder 
+                //Example:  Process everything in the libs/ui folder
                 // "RC png-ui": {
                 //  "pattern": "(.*libs\\\\/ui\\\\/).*\\.png",
                 //  "params": "copy"