Improves AP startup time and CPU Usage

* MaterialBuilder no longer specifies that all textures are critical
* AssetProcessor escalates jobs that are blocking critical jobs from starting
* AssetProcessor (GUI mode only) reserves half jof the CPU cores for
  critical jobs.  When no critical jobs are running, it uses fewer cores
  to give more CPU usage to the editor to maintain framerates.
* User settings like "Skip startup scan" are now project-specific instead
  of global.
* New user setting: Verbose logging (off by default), reduces log spam
  by about 90% when enabled.

  Timing WITH new changes (starting editor with no cache)
  *   42s (  42s total) analyzing
  * 2m45s (3m28s total) "Asset Processor working..."
  * 1m15s (4m44s total) 60fps interactive editor with defaultlevel (meshes still compiling)

Timing WITHOUT the new changes (starting editor with no cache)
  *   42s (  42s total) analyzing (I did not change analyze so this makes sense)
  * 3m34s (4m18s total) "Asset processor working..."
  * 1m22s (5m40s total) 10-20 fps interactive editor with defaultlevel (meshes still compiling)

So about a 45s saving on "Asset Processor Working"
and about 56s total saving with a 60fps editor while it works in background.

Both timings were captured with all other variables hte same, and were
performed on a "hot" file cache, as in, run it once, then clear cache
and immediately run it again to get official timings.    The timing
on a cold cache for the new changes is even more of a difference than
a cold cache without the changes.

Signed-off-by: Nicholas Lawson <[email protected]>

Code/Tools/AssetProcessor/native/assetprocessor.h

@@ -32,6 +32,9 @@
 
 namespace AssetProcessor
 {
+    // global settings for Asset Processor, they usually come from the engine config file
+    // but inidividual gems, projects, and users can override them.
+    constexpr const char* AssetProcessorSettingsKey{ "/Amazon/AssetProcessor/Settings" };
     constexpr const char* DebugChannel = "Debug"; //Use this channel name if you want to write the message to the log file only.
     constexpr const char* ConsoleChannel = "AssetProcessor";// Use this channel name if you want to write the message to both the console and the log file.
     constexpr const char* FENCE_FILE_EXTENSION = "fence"; //fence file extension
@@ -75,10 +78,11 @@ namespace AssetProcessor
     //! This enum stores all the different job escalation values
     enum JobEscalation
     {
-        ProcessAssetRequestSyncEscalation = 200,
-        ProcessAssetRequestStatusEscalation = 150,
-        AssetJobRequestEscalation = 100,
-        Default = 0
+        ProcessAssetRequestSyncEscalation = 200,     //!< Used when the Sync Compile Asset (blocking compile) is requested
+        ProcessAssetRequestStatusEscalation = 150,   //!< Used when the Get Status is called on a specific asset, but not sync compile.
+        AssetJobRequestEscalation = 100,             //!< Used when the user is asking about a specific job, but not syncing
+        CriticalDependencyEscalation = 50,           //!< Used when it is discovered that a critical job depends on this job.
+        DefaultEscalation = 0
     };
     //! This enum stores all the different asset processor status values
     enum AssetProcessorStatus

Code/Tools/AssetProcessor/native/resourcecompiler/RCCommon.cpp

@@ -59,6 +59,14 @@ namespace AssetProcessor
             );
     }
 
+    bool QueueElementID::operator!=(const QueueElementID& other) const
+    {
+        // if this becomes a hotspot in profile, we could use CRCs or other boost to comparison here.  These classes are constructed rarely
+        // compared to how commonly they are compared with each other.
+        return !(operator==(other));
+    }
+
+
     bool QueueElementID::operator<(const QueueElementID& other) const
     {
         int compare = m_sourceAssetReference.AbsolutePath().Compare(other.m_sourceAssetReference.AbsolutePath());

Code/Tools/AssetProcessor/native/resourcecompiler/RCCommon.h

@@ -29,6 +29,7 @@ namespace AssetProcessor
         void SetJobDescriptor(QString jobDescriptor);
         bool operator==(const QueueElementID& other) const;
         bool operator<(const QueueElementID& other) const;
+        bool operator!=(const QueueElementID& other) const;
 
     protected:
         SourceAssetReference m_sourceAssetReference;

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp

@@ -38,6 +38,46 @@ namespace AssetProcessor
         }
     }
 
+    void PrintJob(RCJob* actualJob, int idx)
+    {
+        if (actualJob)
+        {
+            AZ_Printf(
+                AssetProcessor::DebugChannel,
+                "    Job %04i: (Escalation: %i) (Priority: %3i) (Status: %10s) (Crit? %s) (Plat: %s) (MissingDeps? %s) - %s\n",
+                idx,
+                actualJob->JobEscalation(),
+                actualJob->GetPriority(),
+                RCJob::GetStateDescription(actualJob->GetState()).toUtf8().constData(),
+                actualJob->IsCritical() ? "Y" : "N",
+                actualJob->GetPlatformInfo().m_identifier.c_str(),
+                actualJob->HasMissingSourceDependency() ? "Y" : "N",
+                actualJob->GetJobEntry().GetAbsoluteSourcePath().toUtf8().constData());
+
+            for (const JobDependencyInternal& jobDependencyInternal : actualJob->GetJobDependencies())
+            {
+                AZ_Printf(AssetProcessor::DebugChannel, "        Depends on: %s\n",
+                    jobDependencyInternal.ToString().c_str());
+            }
+        }
+    }
+
+    void RCQueueSortModel::DumpJobListInSortOrder()
+    {
+        AZ_Printf(AssetProcessor::DebugChannel, "------------------------------------------------------------\n");
+        AZ_Printf(AssetProcessor::DebugChannel, "RCQueueSortModel: Printing Job list in sorted order:\n");
+        for (int idx = 0; idx < rowCount(); ++idx)
+        {
+            QModelIndex parentIndex = mapToSource(index(idx, 0));
+            RCJob* actualJob = m_sourceModel->getItem(parentIndex.row());
+            PrintJob(actualJob, idx);
+        }
+
+        RCJob* nextJob = GetNextPendingJob();
+        AZ_Printf(AssetProcessor::DebugChannel, "Next job:\n")
+        PrintJob(nextJob, 0);
+    }
+
     RCJob* RCQueueSortModel::GetNextPendingJob()
     {
         if (m_dirtyNeedsResort)
@@ -109,6 +149,12 @@ namespace AssetProcessor
 
                         if (m_sourceModel->isInFlight(elementId) || m_sourceModel->isInQueue(elementId))
                         {
+                            // escalate the job we depend on, if we're critical or escalated ourselves.
+                            if ((actualJob->JobEscalation() != AssetProcessor::DefaultEscalation) || (actualJob->IsCritical()))
+                            {
+                                m_sourceModel->UpdateJobEscalation(elementId, AssetProcessor::CriticalDependencyEscalation);
+                            }
+
                             canProcessJob = false;
                             if (!anyPendingJob || (anyPendingJob->HasMissingSourceDependency() && !actualJob->HasMissingSourceDependency()))
                             {
@@ -169,16 +215,9 @@ namespace AssetProcessor
         // auto fail jobs always take priority to give user feedback asap.
         bool autoFailLeft = leftJob->IsAutoFail();
         bool autoFailRight = rightJob->IsAutoFail();
-        if (autoFailLeft)
+        if (autoFailLeft != autoFailRight)
         {
-            if (!autoFailRight)
-            {
-                return true; // left before right
-            }
-        }
-        else if (autoFailRight)
-        {
-            return false; // right before left.
+            return autoFailLeft;
         }
 
         // Check if either job was marked as having a missing source dependency.
@@ -200,18 +239,10 @@ namespace AssetProcessor
         // Critical jobs should run first, so skip the comparison here if either job is critical, to allow criticality to come in first.
         bool platformsMatch = leftJob->GetPlatformInfo().m_identifier == rightJob->GetPlatformInfo().m_identifier;
 
-        // first thing to check is in platform.
+        // first thing to check is in platform.  If you're currently connected to the editor or other tool on a given platform
+        // you should prioritize those assets.
         if (!platformsMatch)
         {
-            if (leftJob->GetPlatformInfo().m_identifier == AssetBuilderSDK::CommonPlatformName)
-            {
-                return true;
-            }
-            if (rightJob->GetPlatformInfo().m_identifier == AssetBuilderSDK::CommonPlatformName)
-            {
-                return false;
-            }
-
             bool leftActive = m_currentlyConnectedPlatforms.contains(leftJob->GetPlatformInfo().m_identifier.c_str());
             bool rightActive = m_currentlyConnectedPlatforms.contains(rightJob->GetPlatformInfo().m_identifier.c_str());
 
@@ -229,30 +260,21 @@ namespace AssetProcessor
         }
 
         // critical jobs take priority
-        if (leftJob->IsCritical())
-        {
-            if (!rightJob->IsCritical())
-            {
-                return true; // left wins.
-            }
-        }
-        else if (rightJob->IsCritical())
+        if (leftJob->IsCritical() != rightJob->IsCritical())
         {
-            return false; // right wins
+            // one of the two is critical.
+            return leftJob->IsCritical();
         }
 
         int leftJobEscalation = leftJob->JobEscalation();
         int rightJobEscalation = rightJob->JobEscalation();
-
-        // This function, even though its called lessThan(), really is asking, does LEFT come before RIGHT
-        // The higher the escalation, the more important the request, and thus the sooner we want to process the job
-        // Which means if left has a higher escalation number than right, its LESS THAN right.
         if (leftJobEscalation != rightJobEscalation)
         {
             return leftJobEscalation > rightJobEscalation;
         }
 
-        // arbitrarily, lets have PC get done first since pc-format assets are what the editor uses.
+        // arbitrarily, lets prioritize assets for the tools host platform, ie, if you're on a PC, process PC assets before
+        // you process android assets, so that the editor and other tools start quicker.
         if (!platformsMatch)
         {
             if (leftJob->GetPlatformInfo().m_identifier == AzToolsFramework::AssetSystem::GetHostAssetPlatform())

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h

@@ -54,6 +54,9 @@ namespace AssetProcessor
         bool filterAcceptsRow(int source_row, const QModelIndex& source_parent) const override;
         bool lessThan(const QModelIndex& left, const QModelIndex& right) const override;
 
+        // Debugging functions
+        void DumpJobListInSortOrder();
+
     public Q_SLOTS:
         void OnEscalateJobs(AssetProcessor::JobIdEscalationList jobIdEscalationList);
 

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp

@@ -8,33 +8,84 @@
 
 #include "rccontroller.h"
 #include <native/resourcecompiler/RCCommon.h>
+#include <AzCore/std/parallel/thread.h>
 #include <QTimer>
 #include <QThreadPool>
 
-
-
 namespace AssetProcessor
 {
-    RCController::RCController(int cfg_minJobs, int cfg_maxJobs, QObject* parent)
-        : QObject(parent)
-        , m_dispatchingJobs(false)
-        , m_shuttingDown(false)
+    void RCController::UpdateAndComputeJobSlots()
     {
-        AssetProcessorPlatformBus::Handler::BusConnect();
+        if (auto settingsRegistry = AZ::SettingsRegistry::Get())
+        {
+            auto settingsRoot = AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorSettingsKey);
+            AZ::s64 valueFromRegistry = 0;
+            if (settingsRegistry->Get(valueFromRegistry, settingsRoot + "/Jobs/maxJobs"))
+            {
+                m_maxJobs = aznumeric_cast<int>(valueFromRegistry);
+            }
+
+            settingsRegistry->Get(m_alwaysUseMaxJobs, settingsRoot + "/Jobs/AlwaysUseMaxJobs");
+        }
+
+        if (m_maxJobs <= 1) // its not set in the registry to a specific value, or the registry set it to 0 (auto)
+        {
+            // Determine a good starting value for max jobs, we want to use hand tuned numbers for 2, 4, 8, 12, 16, etc
+            unsigned int cpuConcurrency = AZStd::thread::hardware_concurrency();
+            if (cpuConcurrency <= 1)
+            {
+                AZ_Printf(
+                    ConsoleChannel,
+                    "Unable to determine the number of hardware threads supported on this platform, assuming 4.\n",
+                    cpuConcurrency);
+                cpuConcurrency = 4; // we can't query it on this platform, set a reasonable default that gets some work done
+            }
+            AZ_Printf(ConsoleChannel, "Auto (0) selected for maxJobs - auto-configuring based on %u available CPU cores.\n", cpuConcurrency)
 
-        // Determine a good starting value for max jobs
-        int maxJobs = QThread::idealThreadCount();
-        if (maxJobs == -1)
+            // for very low numbers of cores, hand-tune the values, these might be logical cores (hyperthread) and not real ones.
+            // we will reserve about half of this for "backround processing" and then the other half will be reserved for on-demand
+            // (critical or escalated) processing when we actually dispatch jobs.
+            if (cpuConcurrency <= 4)
+            {
+                m_maxJobs = 3;
+            }
+            else if (cpuConcurrency <= 6)
+            {
+                m_maxJobs = 5;
+            }
+            else
+            {
+                // for larger number of cores, 8, 16, 24, we want a few extra cores free
+                m_maxJobs = (cpuConcurrency - 2);
+            }
+        }
+
+        // final fail-safe
+        if (m_maxJobs < 2)
+        {
+            m_maxJobs = 2;
+        }
+        AZ_Printf(ConsoleChannel, "Asset Processor CPU Usage: (settings registry 'Jobs' section):\n")
+        AZ_Printf(ConsoleChannel, "    - Process up to %u jobs in parallel\n", m_maxJobs);
+        if (m_alwaysUseMaxJobs)
+        {
+            AZ_Printf(ConsoleChannel, "    - use all %u jobs whenever possible\n", m_maxJobs);
+        }
+        else
         {
-            maxJobs = 3;
+            AZ_Printf(ConsoleChannel, "    - only use %u jobs when critical work is waiting, %u otherwise.\n", m_maxJobs, AZStd::GetMax(m_maxJobs / 2u, 1u));
         }
 
-        maxJobs = qMax<int>(maxJobs - 1, 1);
+    }
 
-        // if the user has specified max jobs in the cfg file, then we obey their request
-        // regardless of whether they have chosen something bad or not - they would have had to explicitly
-        // pick this value (we ship with default 0 meaning auto), so if they've changed it, they intend it that way
-        m_maxJobs = cfg_maxJobs ? qMax(cfg_minJobs, cfg_maxJobs) :  maxJobs;
+    RCController::RCController(QObject* parent)
+        : QObject(parent)
+        , m_dispatchingJobs(false)
+        , m_shuttingDown(false)
+    {
+        AssetProcessorPlatformBus::Handler::BusConnect();
+
+        UpdateAndComputeJobSlots();
 
         m_RCQueueSortModel.AttachToModel(&m_RCJobListModel);
 
@@ -54,11 +105,6 @@ namespace AssetProcessor
         m_RCQueueSortModel.AttachToModel(nullptr);
     }
 
-    RCJobListModel* RCController::GetQueueModel()
-    {
-        return &m_RCJobListModel;
-    }
-
     void RCController::StartJob(RCJob* rcJob)
     {
         Q_ASSERT(rcJob);
@@ -334,22 +380,42 @@ namespace AssetProcessor
         if (!m_dispatchingJobs)
         {
             m_dispatchingJobs = true;
-            RCJob* rcJob = m_RCQueueSortModel.GetNextPendingJob();
 
-            while (m_RCJobListModel.jobsInFlight() < m_maxJobs && rcJob && !m_shuttingDown)
+            do
             {
-                if (m_dispatchingPaused)
+                RCJob* rcJob = m_RCQueueSortModel.GetNextPendingJob();
+
+                if (!rcJob)
                 {
-                    // note, even if dispatching is "paused" we start all "auto fail jobs" so that user gets instant feedback on failure.
-                    if (!rcJob->IsAutoFail())
+                    // there aren't any jobs remaining to dispatch.
+                    break;
+                }
+
+                // note that critical jobs and escalated jobs will always be at the top of the list
+                bool criticalOrEscalated = rcJob->IsCritical() || (rcJob->JobEscalation() > AssetProcessor::DefaultEscalation);
+
+                // do we have an open slot for this job?
+                unsigned int numJobsInFlight = m_RCJobListModel.jobsInFlight();
+                unsigned int regularJobLimit = m_alwaysUseMaxJobs ? m_maxJobs : AZStd::GetMax(m_maxJobs / 2, 1u);
+                unsigned int maxJobsToStart = criticalOrEscalated ? m_maxJobs : regularJobLimit;
+
+                // note that "auto fail jobs" oimmediately return as failed without doing any processing
+                // so they get to skip the line (they don't use up a thread
+                bool isAutoJob = rcJob->IsAutoFail();
+                bool tooManyJobs = numJobsInFlight >= maxJobsToStart;
+
+                if (!isAutoJob)
+                {
+                    if ((tooManyJobs) || (m_dispatchingPaused))
                     {
+                        // already using too much slots.
                         break;
                     }
                 }
                 StartJob(rcJob);
-                rcJob = m_RCQueueSortModel.GetNextPendingJob();
-            }
-            m_dispatchingJobs = false;
+            } while (true);
+            
+             m_dispatchingJobs = false;
         }
     }
     void RCController::DispatchJobs()
@@ -423,8 +489,9 @@ namespace AssetProcessor
             // PerformHeursticSearch already prints out the results.
             m_RCQueueSortModel.OnEscalateJobs(escalationList);
         }
-        // do not print a warning out when this fails, its fine for things to escalate jobs as a matter of course just to "make sure" they are escalated
-        // and its fine if none are in the build queue.
+
+        // escalating a job could free up an idle cpu thats dedicated to critical or escalated jobs.
+        DispatchJobs();
     }
 
     void RCController::OnEscalateJobsBySourceUUID(QString platform, AZ::Uuid sourceUuid)
@@ -445,6 +512,10 @@ namespace AssetProcessor
         }
         // do not print a warning out when this fails, its fine for things to escalate jobs as a matter of course just to "make sure" they are escalated
         // and its fine if none are in the build queue.
+
+        
+        // escalating a job could free up an idle cpu thats dedicated to critical or escalated jobs.
+        DispatchJobs();
     }
 
     void RCController::OnJobComplete(JobEntry completeEntry, AzToolsFramework::AssetSystem::JobStatus state)

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h

@@ -45,12 +45,9 @@ namespace AssetProcessor
             cmdExecute,
             cmdTerminate
         };
-        RCController() = default;
-        explicit RCController(int minJobs, int maxJobs, QObject* parent = 0);
+        explicit RCController(QObject* parent = 0);
         virtual ~RCController();
 
-        AssetProcessor::RCJobListModel* GetQueueModel();
-
         void StartJob(AssetProcessor::RCJob* rcJob);
         int NumberOfPendingCriticalJobsPerPlatform(QString platform);
 
@@ -105,13 +102,17 @@ namespace AssetProcessor
         void OnJobComplete(JobEntry completeEntry, AzToolsFramework::AssetSystem::JobStatus status);
         void OnAddedToCatalog(JobEntry jobEntry);
 
+        //! The config about # of jobs and slots may have changed, recompute it.
+        void UpdateAndComputeJobSlots();
+
     protected:
         AssetProcessor::RCQueueSortModel m_RCQueueSortModel;
 
     private:
         void FinishJob(AssetProcessor::RCJob* rcJob);
 
-        unsigned int m_maxJobs;
+        unsigned int m_maxJobs = 0; //<! 0 means autocompute, read from registry key
+        bool m_alwaysUseMaxJobs = false; //<! normally, it only uses maxJobs cpu cores when critical or escalated work is present to save CPU usage
 
         bool m_dispatchingJobs = false;
         bool m_shuttingDown = false;

Code/Tools/AssetProcessor/native/resourcecompiler/rcjob.cpp

@@ -97,6 +97,13 @@ namespace AssetProcessor
 
     void RCJob::Init(JobDetails& details)
     {
+        // jobs for the "Common" platform exist to emit additional source files, which themselves could be critical
+        // so they are automatically critical as well.
+        if (GetPlatformInfo().m_identifier == AssetBuilderSDK::CommonPlatformName)
+        {
+            details.m_critical = true;
+        }
+
         m_jobDetails = AZStd::move(details);
         m_queueElementID = QueueElementID(GetJobEntry().m_sourceAssetReference, GetPlatformInfo().m_identifier.c_str(), GetJobKey());
     }

Code/Tools/AssetProcessor/native/resourcecompiler/rcjob.h

@@ -263,7 +263,7 @@ namespace AssetProcessor
 
         QueueElementID m_queueElementID; // cached to prevent lots of construction of this all over the place
 
-        int m_JobEscalation = AssetProcessor::JobEscalation::Default; // Escalation indicates how important the job is and how soon it needs processing, the greater the number the greater the escalation
+        int m_JobEscalation = AssetProcessor::JobEscalation::DefaultEscalation; // Escalation indicates how important the job is and how soon it needs processing, the greater the number the greater the escalation
 
         QDateTime m_timeCreated;
         QDateTime m_timeLaunched;

Code/Tools/AssetProcessor/native/resourcecompiler/rcjoblistmodel.cpp

@@ -95,15 +95,42 @@ namespace AssetProcessor
         return m_finishedJobsNotInCatalog.count();
     }
 
-    void RCJobListModel::UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int jobEscalation)
+    void RCJobListModel::UpdateJobEscalation(const QueueElementID& toEscalate, int valueToEscalateTo)
+    {
+        for (auto foundInQueue = m_jobsInQueueLookup.find(toEscalate); foundInQueue != m_jobsInQueueLookup.end(); ++foundInQueue)
+        {
+            RCJob* job = foundInQueue.value();
+
+            // this is a multi-map, so we have to keep going until it no longer matches.
+            if (!job)
+            {
+                continue;
+            }
+
+            if (job->GetElementID() != toEscalate)
+            {
+                break;
+            }
+
+            if (job->JobEscalation() != valueToEscalateTo)
+            {
+                UpdateJobEscalation(job, valueToEscalateTo);
+            }
+        }
+    }
+
+    void RCJobListModel::UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int valueToEscalateTo)
     {
         for (int idx = 0; idx < rowCount(); ++idx)
         {
             RCJob* job = getItem(idx);
             if (job == rcJob)
             {
-                rcJob->SetJobEscalation(jobEscalation);
-                Q_EMIT dataChanged(index(idx, 0), index(idx, columnCount() - 1));
+                if (job->JobEscalation() != valueToEscalateTo)
+                {
+                    job->SetJobEscalation(valueToEscalateTo);
+                    Q_EMIT dataChanged(index(idx, 0), index(idx, columnCount() - 1));
+                }
                 break;
             }
         }

Code/Tools/AssetProcessor/native/resourcecompiler/rcjoblistmodel.h

@@ -79,7 +79,8 @@ namespace AssetProcessor
         // Returns how many finished jobs that haven't been updated in the catalog.
         unsigned int jobsPendingCatalog() const;
 
-        void UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int jobPrioririty);
+        void UpdateJobEscalation(const QueueElementID& toEscalate, int valueToEscalateTo);
+        void UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int valueToEscalateTo);
         void UpdateRow(int jobIndex);
 
         bool isEmpty();

Code/Tools/AssetProcessor/native/ui/MainWindow.cpp

@@ -599,65 +599,60 @@ void MainWindow::Activate()
     AzQtComponents::CheckBox::applyToggleSwitchStyle(ui->modtimeSkippingCheckBox);
     AzQtComponents::CheckBox::applyToggleSwitchStyle(ui->disableStartupScanCheckBox);
     AzQtComponents::CheckBox::applyToggleSwitchStyle(ui->debugOutputCheckBox);
+    AzQtComponents::CheckBox::applyToggleSwitchStyle(ui->verboseLoggingCheckbox);
 
     const auto apm = m_guiApplicationManager->GetAssetProcessorManager();
 
-    // Note: the settings can't be used in ::MainWindow(), because the application name
-    // hasn't been set up and therefore the settings will load from somewhere different than later
-    // on.
-    // Read the current settings to give command line options a chance to override the default
-    QSettings settings;
-    settings.beginGroup("Options");
-    bool zeroAnalysisModeFromSettings = settings.value("EnableZeroAnalysis", QVariant(true)).toBool() || apm->GetModtimeSkippingFeatureEnabled();
-    bool enableBuilderDebugFlag = settings.value("EnableBuilderDebugFlag", QVariant(false)).toBool() || apm->GetBuilderDebugFlag();
-    bool initialScanSkippingEnabled = settings.value("SkipInitialScan", QVariant(false)).toBool() || apm->GetInitialScanSkippingFeatureEnabled();
-    settings.endGroup();
-
+    // Zero Analysis mode ("Fast scan mode") is enabled by default in the gui (running this mainwindow file), false in batch mode.
+    bool zeroAnalysisMode = AssetUtilities::GetUserSetting("zeroAnalysisMode", true);
+    bool enableBuilderDebugFlag = AssetUtilities::GetUserSetting("enableBuilderDebugFlag", apm->GetBuilderDebugFlag());
+    bool initialScanSkippingEnabled = AssetUtilities::GetUserSetting("initialScanSkippingEnabled", apm->GetInitialScanSkippingFeatureEnabled());
+    bool verboseLogDump = AssetUtilities::GetUserSetting("verboseLogging", false);
     // zero analysis flag
-    apm->SetEnableModtimeSkippingFeature(zeroAnalysisModeFromSettings);
-    ui->modtimeSkippingCheckBox->setCheckState(zeroAnalysisModeFromSettings ? Qt::Checked : Qt::Unchecked);
 
-    // Connect after updating settings to avoid saving a command line override
-    QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
-        [this](int newCheckState)
-    {
-        bool newOption = newCheckState == Qt::Checked ? true : false;
-        m_guiApplicationManager->GetAssetProcessorManager()->SetEnableModtimeSkippingFeature(newOption);
-        QSettings settingsInCallback;
-        settingsInCallback.beginGroup("Options");
-        settingsInCallback.setValue("EnableZeroAnalysis", QVariant(newOption));
-        settingsInCallback.endGroup();
-    });
-
-    // output debug flag
+    apm->SetEnableModtimeSkippingFeature(zeroAnalysisMode);
     apm->SetBuilderDebugFlag(enableBuilderDebugFlag);
+    apm->SetInitialScanSkippingFeature(initialScanSkippingEnabled);
+
+    // first, update the visual checkboxes, and then connect to the "changed" notify
+    // so that we don't save settings applied by the apm command line
+    ui->modtimeSkippingCheckBox->setCheckState(zeroAnalysisMode ? Qt::Checked : Qt::Unchecked);
     ui->debugOutputCheckBox->setCheckState(enableBuilderDebugFlag ? Qt::Checked : Qt::Unchecked);
+    ui->disableStartupScanCheckBox->setCheckState(initialScanSkippingEnabled ? Qt::Checked : Qt::Unchecked);
+    ui->verboseLoggingCheckbox->setCheckState(verboseLogDump ? Qt::Checked : Qt::Unchecked);
+
+    QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
+        [this](int newCheckState)
+        {
+            bool newOption = newCheckState == Qt::Checked ? true : false;
+            m_guiApplicationManager->GetAssetProcessorManager()->SetEnableModtimeSkippingFeature(newOption);
+            AssetUtilities::SetUserSetting("zeroAnalysisMode", newOption);
+        });
 
     QObject::connect(ui->debugOutputCheckBox, &QCheckBox::stateChanged, this,
         [this](int newCheckState)
         {
             bool newOption = newCheckState == Qt::Checked ? true : false;
             m_guiApplicationManager->GetAssetProcessorManager()->SetBuilderDebugFlag(newOption);
-            QSettings settingsInCallback;
-            settingsInCallback.beginGroup("Options");
-            settingsInCallback.setValue("EnableBuilderDebugFlag", QVariant(newOption));
-            settingsInCallback.endGroup();
+            AssetUtilities::SetUserSetting("enableBuilderDebugFlag", newOption);
         });
 
-    apm->SetInitialScanSkippingFeature(initialScanSkippingEnabled);
-    ui->disableStartupScanCheckBox->setCheckState(initialScanSkippingEnabled ? Qt::Checked : Qt::Unchecked);
 
     QObject::connect(ui->disableStartupScanCheckBox, &QCheckBox::stateChanged, this,
         [](int newCheckState)
-    {
-        bool newOption = newCheckState == Qt::Checked ? true : false;
-        // don't change initial scan skipping feature value, as it's only relevant on the first scan
-        // save the value for the next run
-        QSettings settingsInCallback;
-        settingsInCallback.beginGroup("Options");
-        settingsInCallback.setValue("SkipInitialScan", QVariant(newOption));
-        settingsInCallback.endGroup();
-    });
+        {
+            // this is not something that we change while running, so just set it for next time.
+            bool newOption = newCheckState == Qt::Checked ? true : false;
+            AssetUtilities::SetUserSetting("initialScanSkippingEnabled", newOption);
+        });
+
+     QObject::connect(ui->verboseLoggingCheckbox, &QCheckBox::stateChanged, this,
+        [](int newCheckState)
+        {
+            bool newOption = newCheckState == Qt::Checked ? true : false;
+            AssetUtilities::SetUserSetting("verboseLogging", newOption);
+            // Todo: Notify the log system immediately
+        });
 
     // Shared Cache tab:
     SetupAssetServerTab();
@@ -1850,7 +1845,6 @@ void MainWindow::ResetLoggingPanel()
 {
     if (m_loggingPanel)
     {
-        m_loggingPanel->AddLogTab(AzToolsFramework::LogPanel::TabSettings("Debug", "", ""));
         m_loggingPanel->AddLogTab(AzToolsFramework::LogPanel::TabSettings("Messages", "", "", true, true, true, false));
         m_loggingPanel->AddLogTab(AzToolsFramework::LogPanel::TabSettings("Warnings/Errors Only", "", "", false, true, true, false));
     }

Code/Tools/AssetProcessor/native/ui/MainWindow.ui

@@ -314,7 +314,7 @@
        <number>8</number>
       </property>
       <item>
-       <widget class="AzQtComponents::SegmentBar" name="buttonList">
+       <widget class="AzQtComponents::SegmentBar" name="buttonList" native="true">
         <property name="sizePolicy">
          <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
           <horstretch>0</horstretch>
@@ -1537,7 +1537,7 @@
          </layout>
         </widget>
         <widget class="QWidget" name="SettingsPage">
-         <layout class="QVBoxLayout" name="verticalLayout_3">
+         <layout class="QVBoxLayout" name="verticalLayout">
           <item>
            <layout class="QVBoxLayout" name="fullScanLayout">
             <item>
@@ -2095,10 +2095,161 @@
             <property name="orientation">
              <enum>Qt::Vertical</enum>
             </property>
+            <property name="sizeType">
+             <enum>QSizePolicy::Fixed</enum>
+            </property>
             <property name="sizeHint" stdset="0">
              <size>
               <width>16</width>
-              <height>40</height>
+              <height>16</height>
+             </size>
+            </property>
+           </spacer>
+          </item>
+          <item>
+           <layout class="QVBoxLayout" name="logFileLevelLayout">
+            <item>
+             <layout class="QHBoxLayout" name="logFileLevelLayoutLine">
+              <item>
+               <widget class="QLabel" name="logfileLevelHeader">
+                <property name="minimumSize">
+                 <size>
+                  <width>170</width>
+                  <height>0</height>
+                 </size>
+                </property>
+                <property name="baseSize">
+                 <size>
+                  <width>0</width>
+                  <height>0</height>
+                 </size>
+                </property>
+                <property name="font">
+                 <font>
+                  <pointsize>12</pointsize>
+                 </font>
+                </property>
+                <property name="text">
+                 <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:12pt;&quot;&gt;Verbose Logging Output&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                </property>
+               </widget>
+              </item>
+              <item>
+               <spacer name="horizontalSpacer_9">
+                <property name="orientation">
+                 <enum>Qt::Horizontal</enum>
+                </property>
+                <property name="sizeType">
+                 <enum>QSizePolicy::Fixed</enum>
+                </property>
+                <property name="sizeHint" stdset="0">
+                 <size>
+                  <width>20</width>
+                  <height>20</height>
+                 </size>
+                </property>
+               </spacer>
+              </item>
+              <item>
+               <widget class="QCheckBox" name="verboseLoggingCheckbox">
+                <property name="sizePolicy">
+                 <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+                  <horstretch>0</horstretch>
+                  <verstretch>0</verstretch>
+                 </sizepolicy>
+                </property>
+                <property name="minimumSize">
+                 <size>
+                  <width>100</width>
+                  <height>0</height>
+                 </size>
+                </property>
+                <property name="text">
+                 <string/>
+                </property>
+               </widget>
+              </item>
+              <item>
+               <spacer name="horizontalSpacer_10">
+                <property name="orientation">
+                 <enum>Qt::Horizontal</enum>
+                </property>
+                <property name="sizeHint" stdset="0">
+                 <size>
+                  <width>40</width>
+                  <height>20</height>
+                 </size>
+                </property>
+               </spacer>
+              </item>
+             </layout>
+            </item>
+            <item>
+             <widget class="Line" name="line_5">
+              <property name="minimumSize">
+               <size>
+                <width>0</width>
+                <height>0</height>
+               </size>
+              </property>
+              <property name="baseSize">
+               <size>
+                <width>0</width>
+                <height>0</height>
+               </size>
+              </property>
+              <property name="styleSheet">
+               <string notr="true">background-color: #616161;</string>
+              </property>
+              <property name="orientation">
+               <enum>Qt::Horizontal</enum>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <layout class="QHBoxLayout" name="logFileLevelLayoutDescription">
+              <item>
+               <widget class="QLabel" name="logFileLevelDescription">
+                <property name="text">
+                 <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;When enables, outputs debugging data to the Asset Processor log files.  Slows down processing.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+                </property>
+                <property name="scaledContents">
+                 <bool>false</bool>
+                </property>
+                <property name="wordWrap">
+                 <bool>true</bool>
+                </property>
+               </widget>
+              </item>
+              <item>
+               <spacer name="indentSpacer4_2">
+                <property name="orientation">
+                 <enum>Qt::Horizontal</enum>
+                </property>
+                <property name="sizeType">
+                 <enum>QSizePolicy::Fixed</enum>
+                </property>
+                <property name="sizeHint" stdset="0">
+                 <size>
+                  <width>20</width>
+                  <height>20</height>
+                 </size>
+                </property>
+               </spacer>
+              </item>
+             </layout>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <spacer name="gapSpacer5">
+            <property name="orientation">
+             <enum>Qt::Vertical</enum>
+            </property>
+            <property name="sizeHint" stdset="0">
+             <size>
+              <width>16</width>
+              <height>77</height>
              </size>
             </property>
            </spacer>

Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp

@@ -60,6 +60,7 @@ namespace AssetProcessor
     //! we filter the main app logs to only include non-job-thread messages:
     class FilteredLogComponent
         : public AzFramework::LogComponent
+        , public AssetUtilities::AssetProcessorUserSettingsNotificationBus::Handler
     {
     public:
         AZ_CLASS_ALLOCATOR(FilteredLogComponent, AZ::SystemAllocator)
@@ -90,11 +91,40 @@ namespace AssetProcessor
                 return;
             }
 
-            AzFramework::LogComponent::OutputMessage(severity, window, message);
+            if ((m_verboseMode) || (severity >= AzFramework::LogFile::SEV_NORMAL))
+            {
+                AzFramework::LogComponent::OutputMessage(severity, window, message);
+            }
+        }
+
+        void Activate() override
+        {
+            if (auto* registry = AZ::SettingsRegistry::Get())
+            {
+                m_verboseMode = AssetUtilities::GetUserSetting("logVerbosity", false);
+            }
+
+            AssetUtilities::AssetProcessorUserSettingsNotificationBus::Handler::BusConnect();
+            AzFramework::LogComponent::Activate();
+        }
+
+        void Deactivate() override
+        {
+            AzFramework::LogComponent::Deactivate();
+            AssetUtilities::AssetProcessorUserSettingsNotificationBus::Handler::BusDisconnect();
+        }
+
+        void OnSettingChanged(const AZStd::string_view& settingName) override
+        {
+            if (settingName == "logVerbosity")
+            {
+                m_verboseMode = AssetUtilities::GetUserSetting("logVerbosity", false);
+            }
         }
 
     protected:
         bool m_inException = false;
+        bool m_verboseMode = false;
     };
 }
 

Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp

@@ -383,7 +383,7 @@ void ApplicationManagerBase::ConnectAssetCatalog()
 
 void ApplicationManagerBase::InitRCController()
 {
-    m_rcController = new AssetProcessor::RCController(m_platformConfiguration->GetMinJobs(), m_platformConfiguration->GetMaxJobs());
+    m_rcController = new AssetProcessor::RCController();
 
     QObject::connect(m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetToProcess, m_rcController, &AssetProcessor::RCController::JobSubmitted);
     QObject::connect(m_rcController, &AssetProcessor::RCController::FileCompiled, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessed, Qt::UniqueConnection);

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp

@@ -627,8 +627,6 @@ namespace AssetProcessor
 
     PlatformConfiguration::PlatformConfiguration(QObject* pParent)
         : QObject(pParent)
-        , m_minJobs(1)
-        , m_maxJobs(8)
     {
     }
 
@@ -1132,19 +1130,7 @@ namespace AssetProcessor
         AZ::IO::FixedMaxPath engineRoot(AZ::IO::PosixPathSeparator);
         settingsRegistry->Get(engineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder);
         engineRoot = engineRoot.LexicallyNormal(); // Normalize the path to use posix slashes
-
-        AZ::s64 jobCount = m_minJobs;
-        if (settingsRegistry->Get(jobCount, AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorSettingsKey) + "/Jobs/minJobs"))
-        {
-            m_minJobs = aznumeric_cast<int>(jobCount);
-        }
-
-        jobCount = m_maxJobs;
-        if (settingsRegistry->Get(jobCount, AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorSettingsKey) + "/Jobs/maxJobs"))
-        {
-            m_maxJobs = aznumeric_cast<int>(jobCount);
-        }
-
+       
         if (!skipScanFolders)
         {
             AZStd::unordered_map<AZStd::string, AZ::IO::Path> gemNameToPathMap;
@@ -1842,16 +1828,6 @@ namespace AssetProcessor
             });
     }
 
-    int PlatformConfiguration::GetMinJobs() const
-    {
-        return m_minJobs;
-    }
-
-    int PlatformConfiguration::GetMaxJobs() const
-    {
-        return m_maxJobs;
-    }
-
     void PlatformConfiguration::EnableCommonPlatform()
     {
         EnablePlatform(AssetBuilderSDK::PlatformInfo{ AssetBuilderSDK::CommonPlatformName, AZStd::unordered_set<AZStd::string>{ "common" } });

Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h

@@ -36,7 +36,6 @@ namespace AZ
 
 namespace AssetProcessor
 {
-    inline constexpr const char* AssetProcessorSettingsKey{ "/Amazon/AssetProcessor/Settings" };
     inline constexpr const char* AssetProcessorServerKey{ "/O3DE/AssetProcessor/Settings/Server" };
     class PlatformConfiguration;
     class ScanFolderInfo;
@@ -178,10 +177,6 @@ namespace AssetProcessor
 
         void EnablePlatform(const AssetBuilderSDK::PlatformInfo& platform, bool enable = true);
 
-        //! Gets the minumum jobs specified in the configuration file
-        int GetMinJobs() const;
-        int GetMaxJobs() const;
-
         void EnableCommonPlatform();
         void AddIntermediateScanFolder();
 
@@ -332,9 +327,6 @@ namespace AssetProcessor
         AZStd::vector<AzFramework::GemInfo> m_gemInfoList;
         mutable AZ::s64 m_intermediateAssetScanFolderId = -1; // Cached ID for intermediate scanfolder, for quick lookups
 
-        int m_minJobs = 1;
-        int m_maxJobs = 3;
-
         // used only during file read, keeps the total running list of all the enabled platforms from all config files and command lines
         AZStd::vector<AZStd::string> m_tempEnabledPlatforms;
 

Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp

@@ -66,6 +66,9 @@ namespace AssetUtilsInternal
     // This is because Qt has to init random number gen on each thread.
     AZ_THREAD_LOCAL bool g_hasInitializedRandomNumberGenerator = false;
 
+    constexpr AZStd::string_view AssetProcessorUserSetregRelPath = "user/Registry/asset_processor.setreg";
+
+
     // so that even if we do init two seeds at exactly the same msec time, theres still this extra
     // changing number
     static AZStd::atomic_int g_randomNumberSequentialSeed;
@@ -158,11 +161,21 @@ namespace AssetUtilsInternal
         return true;
     }
 
-    static bool DumpAssetProcessorUserSettingsToFile(AZ::SettingsRegistryInterface& settingsRegistry,
-        const AZ::IO::FixedMaxPath& setregPath)
+    static bool DumpAssetProcessorUserSettingsToFile(AZ::SettingsRegistryInterface& settingsRegistry)
     {
-        // The AssetProcessor settings are currently under the Bootstrap object(This may change in the future
-        constexpr AZStd::string_view AssetProcessorUserSettingsRootKey = AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey;
+        using namespace AssetUtilities;
+        AZ::IO::FixedMaxPath setregPath = AZ::Utils::GetProjectPath();
+        setregPath /= AssetProcessorUserSetregRelPath;
+
+        // AssetProcessor saves Bootstrap settings and user settings
+        constexpr AZStd::string_view BootstrapSettingsKey = AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey;
+        constexpr AZStd::string_view UserSettingsKey = AssetProcessorUserSettingsRootKey;
+
+        auto allowedListKey = AZ::SettingsRegistryInterface::FixedValueString(BootstrapSettingsKey) + "/allowed_list";
+        auto branchTokenKey = AZ::SettingsRegistryInterface::FixedValueString(BootstrapSettingsKey) + "/assetProcessor_branch_token";
+        auto generalUserSettingsKey = AZ::SettingsRegistryInterface::FixedValueString(UserSettingsKey);
+
+
         AZStd::string apSettingsJson;
         AZ::IO::ByteContainerStream apSettingsStream(&apSettingsJson);
 
@@ -170,23 +183,16 @@ namespace AssetUtilsInternal
         apDumperSettings.m_prettifyOutput = true;
         AZ_PUSH_DISABLE_WARNING(5233, "-Wunknown-warning-option") // Older versions of MSVC toolchain require to pass constexpr in the
                                                                   // capture. Newer versions issue unused warning
-        apDumperSettings.m_includeFilter = [&AssetProcessorUserSettingsRootKey](AZStd::string_view path)
+        apDumperSettings.m_includeFilter = [&allowedListKey, &branchTokenKey, &generalUserSettingsKey](AZStd::string_view path)
         AZ_POP_DISABLE_WARNING
         {
-            // The AssetUtils only updates the following keys in the registry
-            // Dump them all out to the setreg file
-            auto allowedListKey = AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorUserSettingsRootKey)
-                + "/allowed_list";
-            auto branchTokenKey = AZ::SettingsRegistryInterface::FixedValueString(AssetProcessorUserSettingsRootKey)
-                + "/assetProcessor_branch_token";
             // The objects leading up to the keys to dump must be included in order the keys to be dumped
-            return allowedListKey.starts_with(path.substr(0, allowedListKey.size()))
-                || branchTokenKey.starts_with(path.substr(0, branchTokenKey.size()));
+            return allowedListKey.starts_with(path.substr(0, allowedListKey.size())) ||
+                   branchTokenKey.starts_with(path.substr(0, branchTokenKey.size())) ||
+                   generalUserSettingsKey.starts_with(path.substr(0, generalUserSettingsKey.size()));
         };
-        apDumperSettings.m_jsonPointerPrefix = AssetProcessorUserSettingsRootKey;
 
-        if (AZ::SettingsRegistryMergeUtils::DumpSettingsRegistryToStream(settingsRegistry, AssetProcessorUserSettingsRootKey,
-            apSettingsStream, apDumperSettings))
+        if (AZ::SettingsRegistryMergeUtils::DumpSettingsRegistryToStream(settingsRegistry, "", apSettingsStream, apDumperSettings))
         {
             constexpr const char* AssetProcessorTmpSetreg = "asset_processor.setreg.tmp";
             // Write to a temporary file first before renaming it to the final file location
@@ -219,8 +225,7 @@ namespace AssetUtilsInternal
         }
         else
         {
-            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Dump of AssetProcessor User Settings failed at JSON pointer %.*s \n",
-                aznumeric_cast<int>(AssetProcessorUserSettingsRootKey.size()), AssetProcessorUserSettingsRootKey.data());
+            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Dump of AssetProcessor User Settings failed. \n");
         }
 
         return false;
@@ -229,8 +234,6 @@ namespace AssetUtilsInternal
 
 namespace AssetUtilities
 {
-    constexpr AZStd::string_view AssetProcessorUserSetregRelPath = "user/Registry/asset_processor.setreg";
-
     // do not place Qt objects in global scope, they allocate and refcount threaded data.
     AZ::SettingsRegistryInterface::FixedValueString s_projectPath;
     AZ::SettingsRegistryInterface::FixedValueString s_projectName;
@@ -614,9 +617,6 @@ namespace AssetUtilities
 
     bool WriteAllowedlistToSettingsRegistry(const QStringList& newAllowedList)
     {
-        AZ::IO::FixedMaxPath assetProcessorUserSetregPath = AZ::Utils::GetProjectPath();
-        assetProcessorUserSetregPath /= AssetProcessorUserSetregRelPath;
-
         auto settingsRegistry = AZ::SettingsRegistry::Get();
         if (!settingsRegistry)
         {
@@ -649,7 +649,7 @@ namespace AssetUtilities
         AZStd::string azNewAllowedList{ newAllowedList.join(',').toUtf8().constData() };
         settingsRegistry->Set(allowedListKey, azNewAllowedList);
 
-        return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry, assetProcessorUserSetregPath);
+        return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry);
     }
 
     quint16 ReadListeningPortFromSettingsRegistry(QString initialFolder /*= QString()*/)
@@ -944,9 +944,6 @@ namespace AssetUtilities
 
     bool UpdateBranchToken()
     {
-        AZ::IO::FixedMaxPath assetProcessorUserSetregPath = AZ::Utils::GetProjectPath();
-        assetProcessorUserSetregPath /= AssetProcessorUserSetregRelPath;
-
         AZStd::string appBranchToken;
         AzFramework::ApplicationRequests::Bus::Broadcast(&AzFramework::ApplicationRequests::CalculateBranchTokenForEngineRoot, appBranchToken);
 
@@ -964,21 +961,21 @@ namespace AssetUtilities
             if (appBranchToken == registryBranchToken)
             {
                 // no need to update, branch token match
-                AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Branch token (%s) is already correct in (%s)\n", appBranchToken.c_str(), assetProcessorUserSetregPath.c_str());
+                AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Branch token (%s) is already correct, not updating\n", appBranchToken.c_str());
                 return true;
             }
 
-            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Updating branch token (%s) in (%s)\n", appBranchToken.c_str(), assetProcessorUserSetregPath.c_str());
+            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Updating branch token (%s)\n", appBranchToken.c_str());
         }
         else
         {
-            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Adding branch token (%s) in (%s)\n", appBranchToken.c_str(), assetProcessorUserSetregPath.c_str());
+            AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Adding branch token (%s)\n", appBranchToken.c_str());
         }
 
         // Update Settings Registry with new token
         settingsRegistry->Set(branchTokenKey, appBranchToken);
 
-        return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry, assetProcessorUserSetregPath);
+        return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry);
     }
 
     QString ComputeJobDescription(const AssetProcessor::AssetRecognizer* recognizer)
@@ -1052,6 +1049,61 @@ namespace AssetUtilities
         return ReadJobLogResult::Success;
     }
 
+    template<typename T>
+    bool SetUserSetting(const char* settingName, T value)
+    {
+        if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
+        {
+            constexpr AZStd::string_view UserSettingsKey = AssetProcessorUserSettingsRootKey;
+            auto settingKey = AZ::SettingsRegistryInterface::FixedValueString(UserSettingsKey) + "/" + settingName;
+            bool wasSet = settingsRegistry->Set(settingKey, value);
+            if (wasSet)
+            {
+                AssetProcessorUserSettingsNotificationBus::Broadcast(&AssetProcessorUserSettingsNotificationBus::Events::OnSettingChanged, settingName);
+                return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry);
+            }
+        }
+
+        return false;
+    }
+    template<typename T>
+    T GetUserSetting(const char* settingName, T defaultValue)
+    {
+        T resultValue = defaultValue;
+        if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
+        {
+            constexpr AZStd::string_view UserSettingsKey = AssetProcessorUserSettingsRootKey;
+            auto settingKey = AZ::SettingsRegistryInterface::FixedValueString(UserSettingsKey) + "/" + settingName;
+            settingsRegistry->Get(resultValue, settingKey);
+        }
+        return resultValue;
+    }
+
+    bool SaveSettingsFile()
+    {
+        if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
+        {
+            return AssetUtilsInternal::DumpAssetProcessorUserSettingsToFile(*settingsRegistry);
+        }
+        return false;
+    }
+
+    // we implement these for the types the settings registry allows
+    template bool SetUserSetting<bool>(const char* settingName, bool value);
+    template bool GetUserSetting<bool>(const char* settingName, bool defaultValue);
+
+    template bool SetUserSetting<AZ::s64>(const char* settingName, AZ::s64 value);
+    template AZ::s64 GetUserSetting<AZ::s64>(const char* settingName, AZ::s64 defaultValue);
+
+    template bool SetUserSetting<AZ::u64>(const char* settingName, AZ::u64 value);
+    template AZ::u64 GetUserSetting<AZ::u64>(const char* settingName, AZ::u64 defaultValue);
+
+    template bool SetUserSetting<double>(const char* settingName, double value);
+    template double GetUserSetting<double>(const char* settingName, double defaultValue);
+
+    template bool SetUserSetting<AZStd::string>(const char* settingName, AZStd::string value);
+    template AZStd::string GetUserSetting<AZStd::string>(const char* settingName, AZStd::string defaultValue);
+
     unsigned int GenerateFingerprint(const AssetProcessor::JobDetails& jobDetail)
     {
         // it is assumed that m_fingerprintFilesList contains the original file and all dependencies, and is in a stable order without duplicates

Code/Tools/AssetProcessor/native/utilities/assetUtils.h

@@ -23,6 +23,8 @@
 #include <AzCore/IO/Path/Path.h>
 #include <AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h>
 #include <AssetManager/SourceAssetReference.h>
+#include <AzCore/EBus/Event.h>
+#include <AzCore/std/parallel/mutex.h>
 
 namespace AzToolsFramework
 {
@@ -52,6 +54,37 @@ namespace AssetUtilities
 {
     inline constexpr char ProjectPathOverrideParameter[] = "project-path";
 
+    //! You can read and write any sub-keys to this key and it will be persisted to the project user registry folder:
+    inline constexpr AZStd::string_view AssetProcessorUserSettingsRootKey = "/O3DE/AssetProcessor/UserSettings";
+
+    class AssetProcessorUserSettingsNotifications : public AZ::EBusTraits
+    {
+    public:
+        static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Single; // any number of connected listeners
+        static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::Single; // no addressing used.
+        typedef AZStd::recursive_mutex MutexType; // protect bus addition and removal since listeners can disconnect in threads.
+
+        //! Invoked on SetUserSetting(settingName, ...) and the settingName will be just the name of the setting, not the full registry path.
+        //! If you hanle your own reading and writing to the above sub-keys, consider invoking this yourself.
+        virtual void OnSettingChanged(const AZStd::string_view& settingName) = 0;
+    };
+
+    using AssetProcessorUserSettingsNotificationBus = AZ::EBus<AssetProcessorUserSettingsNotifications>;
+
+    //! Set a named user setting.  The name must be json-path compatible name (avoid dots, slashes, spaces, etc)
+    //! The value must be something that can be written into the settings registry.
+    //! Really only meant to be used for very simple types (ints, strings, bools, etc).  Don't use it for structs.
+    //! You can write your own structs as a sub key of AssetProcessorUserSettingsRootKey and invoke SaveSettingsFile.
+    template<typename T>
+    bool SetUserSetting(const char* settingName, T value);
+
+    //! Gets a named user setting, it will be persisted only for this project, for this user.
+    template<typename T>
+    T GetUserSetting(const char* settingName, T defaultValue);
+
+    //! Save user settings into the default settings file.
+    bool SaveSettingsFile();
+
     //! Set precision fingerprint timestamps will be truncated to avoid mismatches across systems/packaging with different file timestamp precisions
     //! Timestamps default to milliseconds.  A value of 1 will keep the default millisecond precision.  A value of 1000 will reduce the precision to seconds
     void SetTruncateFingerprintTimestamp(int precision);
@@ -389,4 +422,21 @@ namespace AssetUtilities
 
         void AppendLog(AzFramework::LogFile::SeverityLevel severity, const char* window, const char* message);
     };
+
+    extern template bool SetUserSetting<bool>(const char* settingName, bool value);
+    extern template bool GetUserSetting<bool>(const char* settingName, bool defaultValue);
+
+    extern template bool SetUserSetting<AZ::s64>(const char* settingName, AZ::s64 value);
+    extern template AZ::s64 GetUserSetting<AZ::s64>(const char* settingName, AZ::s64 defaultValue);
+
+    extern template bool SetUserSetting<AZ::u64>(const char* settingName, AZ::u64 value);
+    extern template AZ::u64 GetUserSetting<AZ::u64>(const char* settingName, AZ::u64 defaultValue);
+
+    extern template bool SetUserSetting<double>(const char* settingName, double value);
+    extern template double GetUserSetting<double>(const char* settingName, double defaultValue);
+
+    extern template bool SetUserSetting<AZStd::string>(const char* settingName, AZStd::string value);
+    extern template AZStd::string GetUserSetting<AZStd::string>(const char* settingName, AZStd::string defaultValue);
+
+    //! Gets a named user setting, it will be persisted only for this project, for this user.
 } // namespace AssetUtilities

Gems/Atom/RPI/Code/Include/Atom/RPI.Reflect/Asset/AssetUtils.h

@@ -34,11 +34,16 @@ namespace AZ
             ATOM_RPI_REFLECT_API Data::AssetId GetAssetIdForProductPath(const char* productPath, TraceLevel reporting = TraceLevel::Warning, Data::AssetType assetType = Data::s_invalidAssetType);
 
             //! Tries to compile the asset at the given product path.
-            //! This will actively try to compile the asset every time it is called, it won't skip compilation just because the
-            //! asset exists. This should only be used for assets that need to be at their most up-to-date version of themselves
+            //! This will check with the Asset processor whenever it is called, even if the asset is already compiled before.
+            //! It will return instantly if the asset does not exist and cannot be found by the asset processor, or if the
+            //! asset processor is not running or connected.  It only takes time if the asset processor is VERY busy (cpu throttled)
+            //! or if the asset needs to be compiled.
+            //! 
+            //! This should only be used for assets that need to be at their most up-to-date version of themselves
             //! before getting loaded into the engine, as it can take seconds to minutes for this call to return. It is synchronously
             //! asking the Asset Processor to compile the asset, and then blocks until it gets a result. If the AP is busy, it can
             //! take a while to get a result even if the asset is already up-to-date.
+            //! 
             //! In release builds where the AP isn't connected this will immediately return with "Unknown".
             //! @param assetProductFilePath - the relative file path to the product asset (ex: default/models/sphere.azmodel)
             //! @param reporting - the reporting level to use for problems.
@@ -49,6 +54,10 @@ namespace AZ
             //! even when you think the AP is (or will be) connected.
             ATOM_RPI_REFLECT_API bool TryToCompileAsset(const AZStd::string& assetProductFilePath, TraceLevel reporting);
 
+            //! Given an asssetID, ensures that it is compiled and ready to load, if it exists.
+            //! Same caveats as above
+            ATOM_RPI_REFLECT_API bool TryToCompileAsset(const AZ::Data::AssetId& assetId, TraceLevel reporting);
+
             //! Gets an Asset<AssetDataT> reference for a given product file path. This function does not cause the asset to load.
             //! @return a null asset if the asset could not be found.
             template<typename AssetDataT>

Gems/Atom/RPI/Code/Source/RPI.Builders/Material/MaterialBuilder.cpp

@@ -183,18 +183,6 @@ namespace AZ
                 }
             }
 
-            // Assign dependencies from image properties
-            for (const auto& [propertyId, propertyValue] : materialSourceData.GetPropertyValues())
-            {
-                AZ_UNUSED(propertyId);
-
-                if (MaterialUtils::LooksLikeImageFileReference(propertyValue))
-                {
-                    MaterialBuilderUtils::AddPossibleImageDependencies(
-                        materialSourcePath, propertyValue.GetValue<AZStd::string>(), outputJobDescriptor);
-                }
-            }
-
             // Create the output jobs for each platform
             for (const AssetBuilderSDK::PlatformInfo& platformInfo : request.m_enabledPlatforms)
             {
@@ -285,6 +273,8 @@ namespace AZ
                 return;
             }
 
+            MaterialBuilderUtils::AddImageAssetDependenciesToProduct(materialAsset.Get(), jobProduct);
+           
             response.m_outputProducts.emplace_back(AZStd::move(jobProduct));
 
             response.m_resultCode = AssetBuilderSDK::ProcessJobResult_Success;

Gems/Atom/RPI/Code/Source/RPI.Builders/Material/MaterialBuilderUtils.cpp

@@ -10,7 +10,11 @@
 #include <Atom/RPI.Edit/Common/AssetUtils.h>
 #include <Atom/RPI.Edit/Material/MaterialSourceData.h>
 #include <Atom/RPI.Edit/Material/MaterialTypeSourceData.h>
+#include <Atom/RPI.Reflect/Image/StreamingImageAsset.h>
+#include <Atom/RPI.Reflect/Material/MaterialAsset.h>
+#include <Atom/RPI.Reflect/Material/MaterialTypeAsset.h>
 #include <AzCore/Settings/SettingsRegistry.h>
+#include <AzToolsFramework/API/EditorAssetSystemAPI.h>
 
 namespace AZ::RPI::MaterialBuilderUtils
 {
@@ -37,30 +41,62 @@ namespace AZ::RPI::MaterialBuilderUtils
         return jobDescriptor.m_jobDependencyList.emplace_back(AZStd::move(jobDependency));
     }
 
-    void AddPossibleImageDependencies(
-        const AZStd::string& originatingSourceFilePath,
-        const AZStd::string& referencedSourceFilePath,
-        AssetBuilderSDK::JobDescriptor& jobDescriptor)
+    void AddImageAssetDependenciesToProduct(
+        const MaterialPropertiesLayout* propertyLayout,
+        const AZStd::vector<MaterialPropertyValue>& propertyValues,
+        AssetBuilderSDK::JobProduct& product)
     {
-        if (!referencedSourceFilePath.empty())
+        if (!propertyLayout)
         {
-            AZStd::string ext;
-            AzFramework::StringFunc::Path::GetExtension(referencedSourceFilePath.c_str(), ext, false);
-            AZStd::to_upper(ext.begin(), ext.end());
+            return;
+        }
 
-            if (!ext.empty())
+        for (size_t propertyIndex = 0; propertyIndex < propertyLayout->GetPropertyCount(); ++propertyIndex)
+        {
+            auto descriptor = propertyLayout->GetPropertyDescriptor(AZ::RPI::MaterialPropertyIndex{ propertyIndex });
+            if (descriptor->GetDataType() == MaterialPropertyDataType::Image)
             {
-                auto& jobDependency = MaterialBuilderUtils::AddJobDependency(
-                    jobDescriptor,
-                    AssetUtils::ResolvePathReference(originatingSourceFilePath, referencedSourceFilePath),
-                    "Image Compile: " + ext,
-                    {},
-                    { 0 });
-                jobDependency.m_type = AssetBuilderSDK::JobDependencyType::OrderOnce;
+                if (propertyIndex >= propertyValues.size())
+                {
+                    continue; // invalid index, but let's not crash!
+                }
+
+                auto propertyValue = propertyValues[propertyIndex];
+                if (propertyValue.IsValid())
+                {
+                    AZ::Data::Asset<ImageAsset> imageAsset = propertyValue.GetValue<AZ::Data::Asset<ImageAsset>>();
+                    if (imageAsset.GetId().IsValid())
+                    {
+                        // preload images (set to NoLoad to avoid this)
+                        auto loadFlags = AZ::Data::ProductDependencyInfo::CreateFlags(AZ::Data::AssetLoadBehavior::PreLoad);
+                        product.m_dependencies.push_back(AssetBuilderSDK::ProductDependency(imageAsset.GetId(), loadFlags));
+                    }
+                }
             }
         }
     }
 
+    void AddImageAssetDependenciesToProduct(const AZ::RPI::MaterialAsset* materialAsset, AssetBuilderSDK::JobProduct& product)
+    {
+        if (!materialAsset)
+        {
+            return;
+        }
+
+        AddImageAssetDependenciesToProduct(materialAsset->GetMaterialPropertiesLayout(), materialAsset->GetPropertyValues(), product);
+        
+    }
+
+    void AddImageAssetDependenciesToProduct(const AZ::RPI::MaterialTypeAsset* materialTypeAsset, AssetBuilderSDK::JobProduct& product)
+    {
+        if (!materialTypeAsset)
+        {
+            return;
+        }
+
+        AddImageAssetDependenciesToProduct(materialTypeAsset->GetMaterialPropertiesLayout(), materialTypeAsset->GetDefaultPropertyValues(), product);
+    }
+
     void AddFingerprintForDependency(const AZStd::string& path, AssetBuilderSDK::JobDescriptor& jobDescriptor)
     {
         jobDescriptor.m_additionalFingerprintInfo +=

Gems/Atom/RPI/Code/Source/RPI.Builders/Material/MaterialBuilderUtils.h

@@ -14,6 +14,9 @@ namespace AZ
 {
     namespace RPI
     {
+        class MaterialAsset;
+        class MaterialTypeAsset;
+
         namespace MaterialBuilderUtils
         {
             //! @brief configure and register a job dependency with the job descriptor
@@ -33,14 +36,14 @@ namespace AZ
                 const AZStd::vector<AZ::u32>& subIds = {},
                 const bool updateFingerprint = true);
 
-            //! Resolve potential paths and add source and job dependencies for image assets
-            //! @param originatingSourceFilePath The path of the .material or .materialtype file being processed
-            //! @param referencedSourceFilePath The path to the referenced file as it appears in the current file
-            //! @param jobDescriptor Used to update job dependencies
-            void AddPossibleImageDependencies(
-                const AZStd::string& originatingSourceFilePath,
-                const AZStd::string& referencedSourceFilePath,
-                AssetBuilderSDK::JobDescriptor& jobDescriptor);
+            //! Given a material asset that has been fully built and prepared,
+            //! add any image dependencies as pre-load dependencies, to the job being emitted.
+            //! This will cause them to auto preload as part of loading the material, as well as make sure
+            //! they are included in any pak files shipped with the product.
+            void AddImageAssetDependenciesToProduct(const AZ::RPI::MaterialAsset* materialAsset, AssetBuilderSDK::JobProduct& product);
+
+            //! Same as the above overload, but for material TYPE assets.
+            void AddImageAssetDependenciesToProduct(const AZ::RPI::MaterialTypeAsset* materialTypeAsset, AssetBuilderSDK::JobProduct& product);
 
             //! Append a fingerprint value to the job descriptor using the file modification time of the specified file path
             void AddFingerprintForDependency(const AZStd::string& path, AssetBuilderSDK::JobDescriptor& jobDescriptor);

Gems/Atom/RPI/Code/Source/RPI.Builders/Material/MaterialTypeBuilder.cpp

@@ -256,18 +256,6 @@ namespace AZ
                     return true;
                 });
 
-            materialTypeSourceData.EnumerateProperties(
-                [&outputJobDescriptor, &materialTypeSourcePath](const MaterialPropertySourceData* property, const MaterialNameContext&)
-                {
-                    if (property->m_dataType == MaterialPropertyDataType::Image &&
-                        MaterialUtils::LooksLikeImageFileReference(property->m_value))
-                    {
-                        MaterialBuilderUtils::AddPossibleImageDependencies(
-                            materialTypeSourcePath, property->m_value.GetValue<AZStd::string>(), outputJobDescriptor);
-                    }
-                    return true;
-                });
-
             for (const auto& pipelinePair : materialTypeSourceData.m_pipelineData)
             {
                 addFunctorDependencies(pipelinePair.second.m_materialFunctorSourceData);
@@ -898,6 +886,8 @@ namespace AZ
                     return;
                 }
 
+                MaterialBuilderUtils::AddImageAssetDependenciesToProduct(materialTypeAsset.Get(), jobProduct);
+
                 response.m_outputProducts.emplace_back(AZStd::move(jobProduct));
             }
 

Gems/Atom/RPI/Code/Source/RPI.Reflect/Asset/AssetUtils.cpp

@@ -8,6 +8,7 @@
 
 #include <Atom/RPI.Reflect/Asset/AssetUtils.h>
 #include <AzCore/Asset/AssetManagerBus.h>
+#include <AzFramework/Asset/AssetSystemBus.h>
 
 namespace AZ
 {
@@ -53,6 +54,26 @@ namespace AZ
                 return true;
             }
 
+            bool TryToCompileAsset(const AZ::Data::AssetId& assetId, TraceLevel reporting)
+            {
+                AzFramework::AssetSystem::AssetStatus status = AzFramework::AssetSystem::AssetStatus_Unknown;
+                AzFramework::AssetSystemRequestBus::BroadcastResult(status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSyncById, assetId);
+
+                if ((status != AzFramework::AssetSystem::AssetStatus_Compiled) && (status != AzFramework::AssetSystem::AssetStatus_Unknown))
+                {
+                    AssetUtilsInternal::ReportIssue(
+                        reporting,
+                        AZStd::string::format(
+                            "TryToCompileAsset::Could not compile asset '%s', status = %u.",
+                            assetId.ToString<AZStd::string>().c_str(),
+                            static_cast<uint32_t>(status)).c_str());
+
+                    return false;
+                }
+
+                return true;
+            }
+
             Data::AssetId GetAssetIdForProductPath(const char* productPath, TraceLevel reporting, Data::AssetType assetType)
             {
                 // Don't create a new entry in the asset catalog for this asset if it doesn't exist.
@@ -71,9 +92,25 @@ namespace AZ
 
                 if (!assetId.IsValid())
                 {
-                    AZStd::string errorMessage = AZStd::string::format(
-                        "Unable to find product asset '%s'. Has the source asset finished building?", productPath);
-                    AssetUtilsInternal::ReportIssue(reporting, errorMessage.c_str());
+                    // Wait for the asset be compiled if possible.  Note that if AP is not connected, this will return immmediately (0ms)
+                    // and if AP is connected, but, the asset cannot be found or is in an error state, it returns almost immediately (~0ms)
+                    // the only time it actually blocks is if the asset IS found, IS in the queue, in which case it escalates it to the very top
+                    // of the queue and processes it ASAP before returning.
+                    AzFramework::AssetSystem::AssetStatus status = AzFramework::AssetSystem::AssetStatus_Unknown;
+                    AzFramework::AssetSystemRequestBus::BroadcastResult(status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, productPath);
+
+                    if (status == AzFramework::AssetSystem::AssetStatus_Compiled)
+                    {
+                        // success, try again.
+                        Data::AssetCatalogRequestBus::BroadcastResult(assetId, &Data::AssetCatalogRequestBus::Events::GetAssetIdByPath, productPath, assetType, AutoGenerateId);
+                    }
+
+                    if (!assetId.IsValid())
+                    {
+                        AZStd::string errorMessage = AZStd::string::format(
+                            "Unable to find product asset '%s'. Has the source asset finished building?", productPath);
+                        AssetUtilsInternal::ReportIssue(reporting, errorMessage.c_str());
+                    }
                 }
 
                 return assetId;

Gems/Atom/RPI/Code/Source/RPI.Reflect/Model/ModelAsset.cpp

@@ -459,18 +459,9 @@ namespace AZ
                 "Model id " AZ_STRING_FORMAT " not found in asset catalog, using fallback model.\n",
                 AZ_STRING_ARG(asset.GetId().ToFixedString()));
 
-            // Find out if the asset is missing completely or just still processing
-            // and escalate the asset to the top of the list if it's queued.
-            AzFramework::AssetSystem::AssetStatus missingAssetStatus = AzFramework::AssetSystem::AssetStatus::AssetStatus_Unknown;
-            AzFramework::AssetSystemRequestBus::BroadcastResult(
-                missingAssetStatus, &AzFramework::AssetSystem::AssetSystemRequests::GetAssetStatusById, asset.GetId().m_guid);
-
-            if (missingAssetStatus == AzFramework::AssetSystem::AssetStatus::AssetStatus_Queued)
-            {
-                bool sendSucceeded = false;
-                AzFramework::AssetSystemRequestBus::BroadcastResult(
-                    sendSucceeded, &AzFramework::AssetSystem::AssetSystemRequests::EscalateAssetByUuid, asset.GetId().m_guid);
-            }
+            // escalate the asset to the top of the processing list, if it can be (this is a fire and forget message that puts it in the queue
+            // and returns instantly, without waiting). 
+            AzFramework::AssetSystemRequestBus::Broadcast(&AzFramework::AssetSystem::AssetSystemRequests::EscalateAssetByUuid, asset.GetId().m_guid);
 
             // Make sure the default model asset has an entry in the asset catalog so that the asset system will try to load it.
             // Note that we specifically give it a 0-byte size and a non-empty path so that the load will just trivially succeed

Registry/settings.assetprocessorbatch.setreg

@@ -0,0 +1,19 @@
+{
+    // This file is only loaded by the AssetProcessorBatch executable.
+    "Amazon": {
+        "AssetProcessor": {
+            "Settings": {
+                "Jobs": {
+                    // maxJobs 0 means use an automatic amount of CPU cores, detected from hardware.
+                    "maxJobs" : 0,  
+
+                    // AlwaysUseMaxJobs: true means always use maxJobs CPUs at all time.
+                    // The default behavior for AP is to reserve half CPUs for background work
+                    // and half CPUs for escalated (user requested) and critical work.
+                    // Batch is usually used in CLI or automated builds,  so no need.
+                    "AlwaysUseMaxJobs": true
+                }
+            }
+        }
+    }
+}