Add option to disable creating a native window for game launchers (#18093) (#18137)

* Initial implementation of enabling a 'window-less' client for the game launcher project

- Added settings registry key `/O3DE/Atom/Bootstrap/ConsoleMode` to control the creation of a native window on the game client. (Default is false)
- Make sure that the Application Multisample State is set and initialized even if the native window is not created
- Make sure that we don't capture the mouse if there is no native window created (Even if we don't disable mouse capture)
- Add new application type query flag : isConsoleMode
- Add enabling console mode through 3 different ways:
  - '-console-mode' command-line argument
  - '-rhi-null' detection  
  - '-NullRenderer' detection
  - '/O3DE/Atom/RPI/Initialization/NullRenderer'  registry detection
---------
Signed-off-by: Steve Pham <[email protected]>

Code/Framework/AzCore/AzCore/Component/ComponentApplicationBus.h

@@ -39,12 +39,24 @@ namespace AZ
 
     struct ApplicationTypeQuery
     {
+        //! Signals if the application is the Editor.
         bool IsEditor() const;
+
+        //! Signals if the application is the tool application, i.e. AssetProcessor.
         bool IsTool() const;
+
+        //! Signals if the application is a game or server launcher.
         bool IsGame() const;
+
+        //! Signals if the application is running headless (console application with no graphics rendering capability enabled).
         bool IsHeadless() const;
+
+        //! Signals if the application is valid or not. This means the application has not been categorized as any one of Editor, Tool, or Game.
         bool IsValid() const;
 
+        //! Signals if the application is running in console mode where the native client window is not created but still (optionally) supports graphics rendering.
+        bool IsConsoleMode() const;
+
         enum class Masks
         {
             Invalid = 0,
@@ -52,6 +64,7 @@ namespace AZ
             Tool = 1 << 1,
             Game = 1 << 2,
             Headless = 1 << 3,
+            ConsoleMode = 1 << 4,
         };
         Masks m_maskValue = Masks::Invalid;
     };
@@ -62,6 +75,7 @@ namespace AZ
     inline bool ApplicationTypeQuery::IsTool() const { return (m_maskValue & Masks::Tool) == Masks::Tool; }
     inline bool ApplicationTypeQuery::IsGame() const { return (m_maskValue & Masks::Game) == Masks::Game; }
     inline bool ApplicationTypeQuery::IsHeadless() const { return (m_maskValue & Masks::Headless) == Masks::Headless; }
+    inline bool ApplicationTypeQuery::IsConsoleMode() const { return (m_maskValue & Masks::ConsoleMode) == Masks::ConsoleMode; }
     inline bool ApplicationTypeQuery::IsValid() const { return m_maskValue != Masks::Invalid; }
 
     using EntityAddedEvent = AZ::Event<AZ::Entity*>;

Code/Framework/AzFramework/AzFramework/Input/System/InputSystemComponent.cpp

@@ -17,6 +17,7 @@
 #include <AzFramework/Input/Devices/Touch/InputDeviceTouch.h>
 #include <AzFramework/Input/Devices/VirtualKeyboard/InputDeviceVirtualKeyboard.h>
 
+#include <AzCore/Component/ComponentApplicationBus.h>
 #include <AzCore/RTTI/BehaviorContext.h>
 #include <AzCore/Serialization/EditContext.h>
 #include <AzCore/Serialization/SerializeContext.h>
@@ -221,6 +222,16 @@ namespace AzFramework
             settingsRegistry->Get(m_touchEnabled, "/O3DE/InputSystem/TouchEnabled");
             settingsRegistry->Get(m_virtualKeyboardEnabled, "/O3DE/InputSystem/VirtualKeyboardEnabled");
             settingsRegistry->Get(m_captureMouseCursor, "/O3DE/InputSystem/Mouse/CaptureMouseCursor");
+
+            // Check if option to capture the mouse cursor is set or not.
+            bool captureMouseCursor{ true };
+            settingsRegistry->Get(captureMouseCursor, "/O3DE/InputSystem/Mouse/CaptureMouseCursor");
+
+            // Make sure that we only disable capturing the mouse (if specified) when we are running in either headless mode or console mode.
+            AZ::ApplicationTypeQuery appType;
+            AZ::ComponentApplicationBus::Broadcast(&AZ::ComponentApplicationBus::Events::QueryApplicationType, appType);
+
+            m_captureMouseCursor = (captureMouseCursor && (!appType.IsHeadless() && !appType.IsConsoleMode()) || appType.IsEditor());
         }
 
         // Create all enabled input devices

Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp

@@ -92,9 +92,72 @@ namespace AzGameFramework
     {
         m_headless = headless;
     }
+    void GameApplication::SetConsoleModeSupported(bool supported)
+    {
+        m_consoleModeSupported = supported;
+    }
 
     void GameApplication::StartCommon(AZ::Entity* systemEntity)
     {
+        if (!this->m_headless)
+        {
+            bool isConsoleMode{ false };
+
+            // If we are not in headless-mode, its still possible to be in console-only mode,
+            // where a native client window will not be created (on platforms that support 
+            // setting console mode at runtime). There are 2 possible triggers for console mode
+
+            // 1. Settings registry : If the registry setting '/O3DE/Launcher/Bootstrap/ConsoleMode' is specified and set to true
+            if (const auto* settingsRegistry = AZ::SettingsRegistry::Get())
+            {
+                settingsRegistry->Get(isConsoleMode, "/O3DE/Launcher/Bootstrap/ConsoleMode");
+
+                // The null renderer can also be set in the settings registry for the RPI
+                bool isRPINullRenderer{ false };
+                if (settingsRegistry->Get(isRPINullRenderer, "/O3DE/Atom/RPI/Initialization/NullRenderer"))
+                {
+                    if (isRPINullRenderer)
+                    {
+                        isConsoleMode = true;
+                    }
+                }
+            }
+
+            // 2. Either '-console-mode' or 'rhi=null' is specified in the command-line argument
+            const AzFramework::CommandLine* commandLine{ nullptr };
+            constexpr const char* commandSwitchConsoleOnly = "console-mode";
+            constexpr const char* commandSwitchNullRenderer = "NullRenderer";
+            constexpr const char* commandSwitchRhi = "rhi";
+            AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetApplicationCommandLine);
+            AZ_Assert(commandLine, "Unable to query application command line to evaluate console-mode switches.");
+            if (commandLine)
+            {
+                if (commandLine->HasSwitch(commandSwitchConsoleOnly) || commandLine->HasSwitch(commandSwitchNullRenderer))
+                {
+                    isConsoleMode = true;
+                } 
+                else if (size_t switchCount = commandLine->GetNumSwitchValues(commandSwitchRhi); switchCount > 0)
+                {
+                    auto rhiValue = commandLine->GetSwitchValue(commandSwitchRhi, switchCount - 1);
+                    if (rhiValue.compare("null")==0)
+                    {
+                        isConsoleMode = true;
+                    }
+                }
+            }
+
+            AZ_Warning("Launcher", (isConsoleMode && !m_consoleModeSupported), "Console-mode was requested but not supported on this platform\n");
+            if (isConsoleMode && m_consoleModeSupported)
+            {
+                AZ_Info("Launcher", "Console only mode enabled.\n");
+                m_consoleMode = true;
+            }
+            else
+            {
+                m_consoleMode = false;
+            }
+        }
+
         AzFramework::Application::StartCommon(systemEntity);
     }
 
@@ -149,6 +212,10 @@ namespace AzGameFramework
         {
             appType.m_maskValue |= AZ::ApplicationTypeQuery::Masks::Headless;
         }
+        if (m_consoleMode && m_consoleModeSupported)
+        {
+            appType.m_maskValue |= AZ::ApplicationTypeQuery::Masks::ConsoleMode;
+        }
     };
 
 } // namespace AzGameFramework

Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.h

@@ -30,18 +30,20 @@ namespace AzGameFramework
         //////////////////////////////////////////////////////////////////////////
         // AZ::ComponentApplication
         AZ::ComponentTypeList GetRequiredSystemComponents() const override;
-        //////////////////////////////////////////////////////////////////////////
-
-
-        void SetHeadless(bool headless);
-
         void CreateStaticModules(AZStd::vector<AZ::Module*>& outModules) override;
+        //////////////////////////////////////////////////////////////////////////
 
         //////////////////////////////////////////////////////////////////////////
         // AzFramework::ApplicationRequests::Bus
         void QueryApplicationType(AZ::ApplicationTypeQuery& appType) const override;
         //////////////////////////////////////////////////////////////////////////
 
+        //! Set the headless state of the game application. This is determined at compile time
+        void SetHeadless(bool headless);
+
+        //! Set the flag indicating if console-only mode is supported. This is determined based on the platform that supports it.
+        void SetConsoleModeSupported(bool supported);
+
     protected:
 
         //////////////////////////////////////////////////////////////////////////
@@ -54,6 +56,10 @@ namespace AzGameFramework
         //////////////////////////////////////////////////////////////////////////
 
         bool m_headless{ false };
+
+        bool m_consoleModeSupported{ true };
+
+        bool m_consoleMode{ false };
     };
 } // namespace AzGameFramework
 

Code/LauncherUnified/Launcher.cpp

@@ -445,6 +445,12 @@ namespace O3DELauncher
         gameApplication.SetHeadless(false);
 #endif // O3DE_HEADLESS_SERVER
 
+#if AZ_TRAIT_CONSOLE_MODE_SUPPORT
+        gameApplication.SetConsoleModeSupported(true);
+#else
+        gameApplication.SetConsoleModeSupported(false);
+#endif // AZ_TRAIT_CONSOLE_MODE_SUPPORT
+
         // Finally add the "launcher" specialization tag into the Settings Registry
         AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddSpecialization(*settingsRegistry, LauncherFilenameTag);
 

Code/LauncherUnified/Launcher.h

@@ -110,7 +110,4 @@ namespace O3DELauncher
     //! Returns the SettingsRegistry specialization tag
     //! that can be used to load settings for the specific launcher
     const char* GetLauncherTypeSpecialization();
-
-    //! Indicates if the application is running completely headless (no GUI)
-    bool IsHeadless();
 }

Code/LauncherUnified/Platform/Android/Launcher_Traits_Android.h

@@ -10,3 +10,4 @@
 #define AZ_TRAIT_LAUNCHER_LOWER_CASE_PATHS 1
 #define AZ_TRAIT_LAUNCHER_USE_CRY_DYNAMIC_MODULE_HANDLE 1
 #define AZ_TRAIT_SHARED_LIBRARY_FILENAME_FORMAT "%s/lib%s.so"
+#define AZ_TRAIT_CONSOLE_MODE_SUPPORT 0

Code/LauncherUnified/Platform/Linux/Launcher_Traits_Linux.h

@@ -10,3 +10,4 @@
 #define AZ_TRAIT_LAUNCHER_LOWER_CASE_PATHS 0
 #define AZ_TRAIT_LAUNCHER_USE_CRY_DYNAMIC_MODULE_HANDLE 1
 #define AZ_TRAIT_SHARED_LIBRARY_FILENAME_FORMAT "%s/lib%s.so"
+#define AZ_TRAIT_CONSOLE_MODE_SUPPORT 1

Code/LauncherUnified/Platform/Mac/Launcher_Traits_Mac.h

@@ -10,3 +10,4 @@
 #define AZ_TRAIT_LAUNCHER_LOWER_CASE_PATHS 1
 #define AZ_TRAIT_LAUNCHER_USE_CRY_DYNAMIC_MODULE_HANDLE 1
 #define AZ_TRAIT_SHARED_LIBRARY_FILENAME_FORMAT "%s/lib%s.dylib"
+#define AZ_TRAIT_CONSOLE_MODE_SUPPORT 0

Code/LauncherUnified/Platform/Windows/Launcher_Traits_Windows.h

@@ -10,3 +10,4 @@
 #define AZ_TRAIT_LAUNCHER_LOWER_CASE_PATHS 0
 #define AZ_TRAIT_LAUNCHER_USE_CRY_DYNAMIC_MODULE_HANDLE 0
 #define AZ_TRAIT_SHARED_LIBRARY_FILENAME_FORMAT R"(%s\%s.dll)"
+#define AZ_TRAIT_CONSOLE_MODE_SUPPORT 0

Code/LauncherUnified/Platform/iOS/Launcher_Traits_iOS.h

@@ -10,3 +10,4 @@
 #define AZ_TRAIT_LAUNCHER_LOWER_CASE_PATHS 1
 #define AZ_TRAIT_LAUNCHER_USE_CRY_DYNAMIC_MODULE_HANDLE 0
 #define AZ_TRAIT_SHARED_LIBRARY_FILENAME_FORMAT "%s/lib%s.dylib"
+#define AZ_TRAIT_CONSOLE_MODE_SUPPORT 0

Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp

@@ -322,12 +322,24 @@ namespace AZ
             {
                 // Create a native window only if it's a launcher (or standalone)
                 // LY editor create its own window which we can get its handle through AzFramework::WindowSystemNotificationBus::Handler's OnWindowCreated() function
+
+                // Query the application type to determine if this is a headless application
                 AZ::ApplicationTypeQuery appType;
                 ComponentApplicationBus::Broadcast(&AZ::ComponentApplicationBus::Events::QueryApplicationType, appType);
+
                 if (appType.IsHeadless())
                 {
                     m_nativeWindow = nullptr;
                 }
+                else if (appType.IsConsoleMode())
+                {
+                    m_nativeWindow = nullptr;
+
+                    // If we are running without a native window, the application multisamplestate still needs to be set and
+                    // initialized so that the shader's SuperVariant name is set and the scene's render pipelines are re-initialized
+                    AZ::RHI::MultisampleState multisampleState;
+                    AZ::RPI::RPISystemInterface::Get()->SetApplicationMultisampleState(multisampleState);
+                }
                 else if (!appType.IsValid() || appType.IsGame())
                 {
                     // GFX TODO - investigate window creation being part of the GameApplication.