Reduce MPS connectivity errors (#14178)

* Add a new lifecycle event to detect when the command line is processed.
This provides a notification that occurs even later in initialization, after Atom has completed its blocking asset loads and any command-line commands have been processed.

Signed-off-by: Mike Balfour <[email protected]>

* Small improvement to reduce the number of packet drops occurring on level sends.

Signed-off-by: Mike Balfour <[email protected]>

* PR feedback - use exponential backoff and ForceUpdate().

Signed-off-by: Mike Balfour <[email protected]>

* PR Feedback "CommandLineProcessed" -> "LegacyCommandLineProcessed"

Signed-off-by: Mike Balfour <[email protected]>

Signed-off-by: Mike Balfour <[email protected]>

Code/Editor/CryEdit.cpp

@@ -1833,6 +1833,11 @@ bool CCryEditApp::InitInstance()
         return true;
     }
 
+    if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
+    {
+        AZ::ComponentApplicationLifecycle::SignalEvent(*settingsRegistry, "LegacyCommandLineProcessed", R"({})");
+    }
+
     if (IsInRegularEditorMode())
     {
         int startUpMacroIndex = GetIEditor()->GetToolBoxManager()->GetMacroIndex("startup", true);

Code/LauncherUnified/Launcher.cpp

@@ -571,6 +571,8 @@ namespace O3DELauncher
 
                 gEnv->pSystem->ExecuteCommandLine(false);
 
+                AZ::ComponentApplicationLifecycle::SignalEvent(*settingsRegistry, "LegacyCommandLineProcessed", R"({})");
+
                 // Run the main loop
                 RunMainLoop(gameApplication);
             }

Code/Tools/AssetProcessor/native/tests/AssetProcessorTest.h

@@ -71,6 +71,7 @@ namespace AssetProcessor
                 AZ::ComponentApplicationLifecycle::RegisterEvent(*settingsRegistry, "FileIOUnavailable");
                 AZ::ComponentApplicationLifecycle::RegisterEvent(*settingsRegistry, "LegacySystemInterfaceCreated");
                 AZ::ComponentApplicationLifecycle::RegisterEvent(*settingsRegistry, "CriticalAssetsCompiled");
+                AZ::ComponentApplicationLifecycle::RegisterEvent(*settingsRegistry, "LegacyCommandLineProcessed");
             }
         }
 

Gems/Multiplayer/Code/Source/Editor/MultiplayerEditorConnection.cpp

@@ -38,8 +38,10 @@ namespace Multiplayer
             editorInterfaceName, ProtocolType::Tcp, TrustZone::ExternalClientToServer, *this);
         m_networkEditorInterface->SetTimeoutMs(AZ::Time::ZeroTimeMs); // Disable timeouts on this network interface
 
-        // Wait to activate the editor-server until LegacySystemInterfaceCreated so that the logging system is ready
-        // Automated testing listens for these logs
+        // Wait to activate the editor-server until:
+        // - LegacySystemInterfaceCreated is signaled, so that the logging system is ready. Automated testing listens for these logs.
+        // - LegacyCommandLineProcessed is signaled, so that everything has initialized and finished their blocking loads, so that it
+        // should be relatively safe to start receiving packets without as much fear of too much time passing between system ticks.
         if (editorsv_isDedicated)
         {
             // Server logs will be piped to the editor so turn off buffering,
@@ -59,7 +61,7 @@ namespace Multiplayer
                     {
                         ActivateDedicatedEditorServer();
                     },
-                    "CriticalAssetsCompiled");
+                    "LegacyCommandLineProcessed");
             }
         }
     }

Gems/Multiplayer/Code/Source/Editor/MultiplayerEditorSystemComponent.cpp

@@ -24,6 +24,7 @@
 #include <AzCore/Console/IConsole.h>
 #include <AzCore/Interface/Interface.h>
 #include <AzCore/Serialization/SerializeContext.h>
+#include <AzCore/std/chrono/chrono.h>
 #include <AzCore/Utils/Utils.h>
 #include <AzFramework/API/ApplicationAPI.h>
 #include <AzNetworking/Framework/INetworking.h>
@@ -410,7 +411,27 @@ namespace Multiplayer
                 editorServerLevelDataPacket.SetLastUpdate(true);
             }
 
-            connection->SendReliablePacket(editorServerLevelDataPacket);
+            // Try to send the packet to the Editor server. Retry if necessary.
+            bool packetSent = false;
+            static constexpr int MaxRetries = 20;
+            int millisecondDelayPerRetry = 10;
+            int numRetries = 0;
+            while (!packetSent && (numRetries < MaxRetries))
+            {
+                packetSent = connection->SendReliablePacket(editorServerLevelDataPacket);
+                if (!packetSent)
+                {
+                    AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(millisecondDelayPerRetry));
+                    numRetries++;
+
+                    // Keep doubling the time between retries up to 1 second, then clamp it there.
+                    millisecondDelayPerRetry = AZStd::min(millisecondDelayPerRetry * 2, 1000);
+
+                    // Force the networking buffers to try and flush before sending the packet again.
+                    AZ::Interface<AzNetworking::INetworking>::Get()->ForceUpdate();
+                }
+            }
+            AZ_Assert(packetSent, "Failed to send level packet after %d tries. Server will fail to run the level correctly.", numRetries);
         }
     }
 

Registry/application_lifecycle_events.setreg

@@ -24,7 +24,8 @@
                 "FileIOAvailable": {},
                 "FileIOUnavailable": {},
                 "LegacySystemInterfaceCreated": {},
-                "CriticalAssetsCompiled": {}
+                "CriticalAssetsCompiled": {},
+                "LegacyCommandLineProcessed": {}
             }
         }
     }