Python Relocation (#17540)

Python update and relocation

* Relocates the 3rd Party Python Package
  * The Python 3rd Party Binary was originally downloaded to an internal `python/runtime/<package-name>` folder instead of the location of the other 3rd party binaries `${HOME}/.o3de/3rdParty`
  * For immutable installations like SNAP, this makes it impossible to import and additional python modules after the fact
  * Python commands were run directly from the binary folder. This change introduces running from a engine location specific virtual environment (under `${HOME}/.o3de/3rdParty/venv/{id}`) where **id** is computed from the engine path.
* Updates the python 3rd Party Package to 3.10.13
  *  Updates python to the latest 3.10 subversion for the latest security patches.
* Updates to references for the version of Python (3.10.5->3.10.13)
* Update AZ::DynamicModuleHandle to add support for loading general shared libraries with the option of enabling global symbols on systems that support it, and add the ability to disable platform-specific filenaming support.
* Add a new support method 'Get3rdPartyDirectory' in AZ::Utils
* Introducing a new PythonLoader to support runtime python operations
  * Add a PAL Trait gated operation in the constructor to perform loading of the python shared library into the global symbols space (see updates to AZ::DynamicModuleHandle)
  * Provides path resolution for python environment related paths
* Updates to the python bootstrap (get_python) to support the new location and workflow
* Updates to the python script and EditorPythonBindings to use the python venv instead
* Misc python2->python3 fixes in legacy scripts

Signed-off-by: Steve Pham <[email protected]>
Co-authored-by: Alex Peterson <[email protected]>
Co-authored-by: lumberyard-employee-dm <[email protected]>

AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/README.txt

@@ -15,7 +15,7 @@ editor tests.
 REQUIREMENTS
 ------------
 
- * Python 3.10.5 (64-bit)
+ * Python 3.10.13 (64-bit)
 
 It is recommended that you completely remove any other versions of Python
 installed on your system.

Code/Framework/AzCore/AzCore/Module/DynamicModuleHandle.cpp

@@ -16,9 +16,9 @@ namespace AZ
     {
     }
 
-    bool DynamicModuleHandle::Load(bool isInitializeFunctionRequired)
+    bool DynamicModuleHandle::Load(bool isInitializeFunctionRequired, bool globalSymbols /*= false*/)
     {
-        LoadStatus status = LoadModule();
+        LoadStatus status = LoadModule(globalSymbols);
         switch (status)
         {
             case LoadStatus::LoadFailure:

Code/Framework/AzCore/AzCore/Module/DynamicModuleHandle.h

@@ -31,16 +31,23 @@ namespace AZ
         DynamicModuleHandle& operator=(const DynamicModuleHandle&) = delete;
 
         /// Creates a platform-specific DynamicModuleHandle.
+        /// \param fullFileName         The file name of the dynamic module to load
+        /// \param correctModuleName    Option to correct the filename to conform to the current platform's dynamic module naming convention. 
+        ///                             (i.e. lib<ModuleName>.so on unix-like platforms)
+        ///
         /// Note that the specified module is not loaded until \ref Load is called.
-        static AZStd::unique_ptr<DynamicModuleHandle> Create(const char* fullFileName);
+        /// \return Unique ptr to the newly created dynamic module handler.
+        static AZStd::unique_ptr<DynamicModuleHandle> Create(const char* fullFileName, bool correctModuleName = true);
 
         /// Loads the module.
         /// Invokes the \ref InitializeDynamicModuleFunction if it is found in the module and this is the first time loading the module.
         /// \param isInitializeFunctionRequired Whether a missing \ref InitializeDynamicModuleFunction
         ///                                     causes the Load to fail.
+        /// \param globalSymbols                On platforms that support it, make the module's symbols global and available for the relocation processing of other modules. Otherwise, the symbols 
+        ///                                     need to be queried manually.
         ///
         /// \return True if the module loaded successfully.
-        bool Load(bool isInitializeFunctionRequired);
+        bool Load(bool isInitializeFunctionRequired, bool globalSymbols = false);
 
         /// Unload the module.
         /// Invokes the \ref UninitializeDynamicModuleFunction if it is found in the module and
@@ -81,7 +88,7 @@ namespace AZ
         };
 
         // Attempt to load a module.
-        virtual LoadStatus LoadModule() = 0;
+        virtual LoadStatus LoadModule(bool globalSymbols) = 0;
         virtual bool       UnloadModule() = 0;
         virtual void*      GetFunctionAddress(const char* functionName) const = 0;
 

Code/Framework/AzCore/AzCore/Utils/Utils.cpp

@@ -15,6 +15,7 @@
 #include <AzCore/IO/GenericStreams.h>
 #include <AzCore/IO/SystemFile.h>
 #include <AzCore/IO/Path/Path.h>
+#include <AzCore/Serialization/Json/JsonUtils.h>
 #include <AzCore/Settings/SettingsRegistry.h>
 #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzCore/StringFunc/StringFunc.h>
@@ -410,6 +411,23 @@ namespace AZ::Utils
         return AZ::Success(AZStd::move(fileContent));
     }
 
+    AZ::Outcome<AZStd::string, AZStd::string> Get3rdPartyDirectory(AZ::SettingsRegistryInterface* settingsRegistry /*= nullptr*/)
+    {
+        // Locate the manifest file
+        auto manifestPath = GetO3deManifestPath(settingsRegistry);
+
+        auto readJsonDocOutput = AZ::JsonSerializationUtils::ReadJsonFile(manifestPath);
+        if (!readJsonDocOutput.IsSuccess())
+        {
+            return AZ::Failure(readJsonDocOutput.GetError());
+        }
+
+        rapidjson::Document configurationFile = readJsonDocOutput.TakeValue();
+
+        auto thirdPartyFolder = configurationFile["default_third_party_folder"].GetString();
+        return AZ::Success(thirdPartyFolder);
+    }
+
     template AZ::Outcome<AZStd::string, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<int8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<uint8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);

Code/Framework/AzCore/AzCore/Utils/Utils.h

@@ -168,6 +168,15 @@ namespace AZ
         AZ::Outcome<Container, AZStd::string> ReadFile(
             AZStd::string_view filePath, size_t maxFileSize = AZStd::numeric_limits<size_t>::max());
 
+
+        //! Retrieves the full path where the 3rd Party path is configured to based on the value of 'default_third_party_folder'
+        //! in the o3de manifest file (o3de_manifest.json)
+        //! @param settingsRegistry pointer to the SettingsRegistry to use for lookup of the manifest file to base the lookup from
+        //! If nullptr, the AZ::Interface instance of the SettingsRegistry is used
+        //! Returns the outcome of the request, the configured 3rd Party path if successful, the error message if not.
+        AZ::Outcome<AZStd::string, AZStd::string> Get3rdPartyDirectory(AZ::SettingsRegistryInterface* settingsRegistry = nullptr);
+
+
         //! error code value returned when GetEnv fails
         enum class GetEnvErrorCode
         {

Code/Framework/AzCore/Platform/Android/AzCore/Module/DynamicModuleHandle_Android.cpp

@@ -17,10 +17,11 @@ namespace AZ::Platform
         return {};
     }
 
-    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool&)
+    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool&, bool globalSymbols)
     {
         // Android 19 does not have RTLD_NOLOAD but it should be OK since only the Editor expects to reopen modules
-        return dlopen(fileName.c_str(), RTLD_NOW);
+        int openFlags = globalSymbols ? RTLD_NOW | RTLD_GLOBAL : RTLD_NOW;
+        return dlopen(fileName.c_str(), openFlags);
     }
 
     void ConstructModuleFullFileName(AZ::IO::FixedMaxPath&)

Code/Framework/AzCore/Platform/Common/Apple/AzCore/Module/DynamicModuleHandle_Apple.cpp

@@ -17,13 +17,14 @@ namespace AZ::Platform
         return AZ::Utils::GetExecutableDirectory();
     }
 
-    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen)
+    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen, bool globalSymbols)
     {
         void* handle = dlopen(fileName.c_str(), RTLD_NOLOAD);
         alreadyOpen = (handle != nullptr);
         if (!alreadyOpen)
         {
-            handle = dlopen(fileName.c_str(), RTLD_NOW);
+            int openFlags = globalSymbols ? RTLD_NOW | RTLD_GLOBAL : RTLD_NOW;
+            handle = dlopen(fileName.c_str(), openFlags);
         }
         return handle;
     }

Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Module/DynamicModuleHandle_UnixLike.cpp

@@ -19,7 +19,7 @@
 namespace AZ::Platform
 {
     AZ::IO::FixedMaxPath GetModulePath();
-    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen);
+    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen, bool globalSymbols);
     void ConstructModuleFullFileName(AZ::IO::FixedMaxPath& fullPath);
     AZ::IO::FixedMaxPath CreateFrameworkModulePath(const AZ::IO::PathView& moduleName);
 }
@@ -32,26 +32,29 @@ namespace AZ
     public:
         AZ_CLASS_ALLOCATOR(DynamicModuleHandleUnixLike, OSAllocator)
 
-        DynamicModuleHandleUnixLike(const char* fullFileName)
+        DynamicModuleHandleUnixLike(const char* fullFileName, bool correctModuleName)
             : DynamicModuleHandle(fullFileName)
             , m_handle(nullptr)
         {
             AZ::IO::FixedMaxPath fullFilePath(AZStd::string_view{m_fileName});
-            if (fullFilePath.HasFilename())
+            if (correctModuleName)
             {
-                AZ::IO::FixedMaxPathString fileNamePath{fullFilePath.Filename().Native()};
-                if (!fileNamePath.starts_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_PREFIX))
+                if (fullFilePath.HasFilename())
                 {
-                    fileNamePath = AZ_TRAIT_OS_DYNAMIC_LIBRARY_PREFIX + fileNamePath;
-                }
+                    AZ::IO::FixedMaxPathString fileNamePath{fullFilePath.Filename().Native()};
+                    if (!fileNamePath.starts_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_PREFIX))
+                    {
+                        fileNamePath = AZ_TRAIT_OS_DYNAMIC_LIBRARY_PREFIX + fileNamePath;
+                    }
 
-                if (!fileNamePath.ends_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION))
-                {
-                    fileNamePath += AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION;
-                }
+                    if (!fileNamePath.ends_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION))
+                    {
+                        fileNamePath += AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION;
+                    }
 
-                fullFilePath.ReplaceFilename(AZStd::string_view(fileNamePath));
-                m_fileName.assign(fullFilePath.Native().data(), fullFilePath.Native().size());
+                    fullFilePath.ReplaceFilename(AZStd::string_view(fileNamePath));
+                    m_fileName.assign(fullFilePath.Native().data(), fullFilePath.Native().size());
+                }
             }
 
             Platform::ConstructModuleFullFileName(fullFilePath);
@@ -129,12 +132,12 @@ namespace AZ
             Unload();
         }
 
-        LoadStatus LoadModule() override
+        LoadStatus LoadModule(bool globalSymbols) override
         {
             AZ::Debug::Trace::Instance().Printf("Module", "Attempting to load module:%s\n", m_fileName.c_str());
             bool alreadyOpen = false;
 
-            m_handle = Platform::OpenModule(m_fileName, alreadyOpen);
+            m_handle = Platform::OpenModule(m_fileName, alreadyOpen, globalSymbols);
 
             if (m_handle)
             {
@@ -188,8 +191,8 @@ namespace AZ
     };
 
     // Implement the module creation function
-    AZStd::unique_ptr<DynamicModuleHandle> DynamicModuleHandle::Create(const char* fullFileName)
+    AZStd::unique_ptr<DynamicModuleHandle> DynamicModuleHandle::Create(const char* fullFileName, bool correctModuleName /*= true*/)
     {
-        return AZStd::unique_ptr<DynamicModuleHandle>(aznew DynamicModuleHandleUnixLike(fullFileName));
+        return AZStd::unique_ptr<DynamicModuleHandle>(aznew DynamicModuleHandleUnixLike(fullFileName, correctModuleName));
     }
 } // namespace AZ

Code/Framework/AzCore/Platform/Common/WinAPI/AzCore/Module/DynamicModuleHandle_WinAPI.cpp

@@ -20,15 +20,18 @@ namespace AZ
         : public DynamicModuleHandle
     {
     public:
-        DynamicModuleHandleWindows(const char* fullFileName)
+        DynamicModuleHandleWindows(const char* fullFileName, bool correctModuleName)
             : DynamicModuleHandle(fullFileName)
             , m_handle(nullptr)
         {
-            // Ensure filename ends in ".dll"
-            // Otherwise filenames like "gem.1.0.0" fail to load (.0 is assumed to be the extension).
-            if (!m_fileName.ends_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION))
+            if (correctModuleName)
             {
-                m_fileName += AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION;
+                // Ensure filename ends in ".dll"
+                // Otherwise filenames like "gem.1.0.0" fail to load (.0 is assumed to be the extension).
+                if (!m_fileName.ends_with(AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION))
+                {
+                    m_fileName += AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION;
+                }
             }
 
             AZ::IO::PathView modulePathView{ m_fileName };
@@ -87,7 +90,7 @@ namespace AZ
             Unload();
         }
 
-        LoadStatus LoadModule() override
+        LoadStatus LoadModule([[maybe_unused]] bool globalSymbols) override
         {
             if (IsLoaded())
             {
@@ -103,6 +106,8 @@ namespace AZ
             {
                 // If module already open, return false, it was not loaded.
                 alreadyLoaded = NULL != GetModuleHandleW(fileNameW);
+
+                // Note: Windows LoadLibrary has no concept of specifying that the module symbols are global or local
                 m_handle = LoadLibraryW(fileNameW);
             }
             else
@@ -154,8 +159,8 @@ namespace AZ
     };
 
     // Implement the module creation function
-    AZStd::unique_ptr<DynamicModuleHandle> DynamicModuleHandle::Create(const char* fullFileName)
+    AZStd::unique_ptr<DynamicModuleHandle> DynamicModuleHandle::Create(const char* fullFileName, bool correctModuleName /*= true*/)
     {
-        return AZStd::unique_ptr<DynamicModuleHandle>(new DynamicModuleHandleWindows(fullFileName));
+        return AZStd::unique_ptr<DynamicModuleHandle>(new DynamicModuleHandleWindows(fullFileName, correctModuleName));
     }
 } // namespace AZ

Code/Framework/AzCore/Platform/Linux/AzCore/Module/DynamicModuleHandle_Linux.cpp

@@ -17,13 +17,14 @@ namespace AZ::Platform
         return AZ::Utils::GetExecutableDirectory();
     }
 
-    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen)
+    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen, bool globalSymbols)
     {
         void* handle = dlopen(fileName.c_str(), RTLD_NOLOAD);
         alreadyOpen = (handle != nullptr);
         if (!alreadyOpen)
         {
-            handle = dlopen(fileName.c_str(), RTLD_NOW);
+            int openFlags = globalSymbols ? RTLD_NOW | RTLD_GLOBAL : RTLD_NOW;
+            handle = dlopen(fileName.c_str(), openFlags);
         }
         return handle;
     }

Code/Framework/AzCore/Platform/iOS/AzCore/Module/DynamicModuleHandle_iOS.cpp

@@ -18,13 +18,14 @@ namespace AZ::Platform
         return AZ::IO::FixedMaxPath(AZ::Utils::GetExecutableDirectory()) / "Frameworks";
     }
 
-    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen)
+    void* OpenModule(const AZ::IO::FixedMaxPathString& fileName, bool& alreadyOpen, bool globalSymbols)
     {
         void* handle = dlopen(fileName.c_str(), RTLD_NOLOAD);
         alreadyOpen = (handle != nullptr);
         if (!alreadyOpen)
         {
-            handle = dlopen(fileName.c_str(), RTLD_NOW);
+            int openFlags = globalSymbols ? RTLD_NOW | RTLD_GLOBAL : RTLD_NOW;
+            handle = dlopen(fileName.c_str(), openFlags);
         }
         return handle;
     }

Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.cpp

@@ -0,0 +1,192 @@
+/*
+* Copyright (c) Contributors to the Open 3D Engine Project.
+* For complete copyright and license terms please see the LICENSE at the root of this distribution.
+*
+* SPDX-License-Identifier: Apache-2.0 OR MIT
+*
+*/
+
+#include <AzToolsFramework/API/PythonLoader.h>
+#include <AzToolsFramework_Traits_Platform.h>
+#include <AzCore/Component/ComponentApplicationBus.h>
+#include <AzCore/IO/FileIO.h>
+#include <AzCore/IO/GenericStreams.h>
+#include <AzCore/IO/Path/Path.h>
+#include <AzCore/Math/Sha1.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/string/string.h>
+#include <AzCore/std/string/string_view.h>
+#include <AzCore/std/string/conversions.h>
+#include <AzCore/std/string/tokenize.h>
+#include <AzCore/Settings/ConfigParser.h>
+#include <AzCore/Utils/Utils.h>
+#include <AzFramework/IO/LocalFileIO.h>
+
+namespace AzToolsFramework::EmbeddedPython
+{
+    PythonLoader::PythonLoader()
+    {
+        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
+        // PYTHON_SHARED_LIBRARY_PATH must be defined in the build scripts and referencing the path to the python shared library
+        #if !defined(PYTHON_SHARED_LIBRARY_PATH)
+        #error "PYTHON_SHARED_LIBRARY_PATH is not defined"
+        #endif
+
+        // Construct the path to the shared python library within the venv folder
+        AZ::IO::FixedMaxPath engineRoot = AZ::IO::FixedMaxPath(AZ::Utils::GetEnginePath());
+        AZ::IO::FixedMaxPath thirdPartyRoot = PythonLoader::GetDefault3rdPartyPath(false);
+        AZ::IO::FixedMaxPath pythonVenvPath = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot);
+
+        AZ::IO::PathView libPythonName = AZ::IO::PathView(PYTHON_SHARED_LIBRARY_PATH).Filename();
+        AZ::IO::FixedMaxPath pythonVenvLibPath = pythonVenvPath / "lib" / libPythonName;
+
+        m_embeddedLibPythonModuleHandle = AZ::DynamicModuleHandle::Create(pythonVenvLibPath.StringAsPosix().c_str(), false);
+        bool loadResult = m_embeddedLibPythonModuleHandle->Load(false, true);
+        AZ_Error("PythonLoader", loadResult, "Failed to load %s.\n", libPythonName.StringAsPosix().c_str());
+        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
+    }
+
+    PythonLoader::~PythonLoader()
+    {
+        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
+        AZ_Assert(m_embeddedLibPythonModuleHandle, "DynamicModuleHandle for python was not created");
+        m_embeddedLibPythonModuleHandle->Unload();
+        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
+    }
+
+    AZ::IO::FixedMaxPath PythonLoader::GetDefault3rdPartyPath(bool createOnDemand)
+    {
+        AZ::IO::FixedMaxPath thirdPartyEnvPathPath;
+
+        // The highest priority for the 3rd party path is the environment variable 'LY_3RDPARTY_PATH'
+        static constexpr const char* env3rdPartyKey = "LY_3RDPARTY_PATH";
+        char env3rdPartyPath[AZ::IO::MaxPathLength] = { '\0' };
+        auto envOutcome = AZ::Utils::GetEnv(AZStd::span(env3rdPartyPath), env3rdPartyKey);
+        if (envOutcome && (strlen(env3rdPartyPath) > 0))
+        {
+            // If so, then use the path that is set as the third party path
+            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(env3rdPartyPath).LexicallyNormal();
+        }
+        // The next priority is to read the 3rd party directory from the manifest file
+        else if (auto manifest3rdPartyResult = AZ::Utils::Get3rdPartyDirectory(); manifest3rdPartyResult.IsSuccess())
+        {
+            thirdPartyEnvPathPath = manifest3rdPartyResult.GetValue();
+        }
+        // Fallback to the default 3rd Party path based on the location of the manifest folder
+        else
+        {
+            auto manifestPath = AZ::Utils::GetO3deManifestDirectory();
+            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(manifestPath) / "3rdParty";
+        }
+
+        if ((!AZ::IO::SystemFile::IsDirectory(thirdPartyEnvPathPath.c_str())) && createOnDemand)
+        {
+            auto createPathResult = AZ::IO::SystemFile::CreateDir(thirdPartyEnvPathPath.c_str());
+            AZ_Assert(createPathResult, "Unable to create missing 3rd Party Folder '%s'", thirdPartyEnvPathPath.c_str())
+        }
+        return thirdPartyEnvPathPath;
+    }
+
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonHomePath(AZ::IO::PathView engineRoot)
+    {
+        AZ::IO::FixedMaxPath thirdPartyFolder = GetDefault3rdPartyPath(true);
+
+        // The python HOME path relative to the executable depends on the host platform the package is created for
+        #if AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot).ParentPath();
+        #else
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot);
+        #endif // AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
+
+        return pythonHomePath;
+    }
+
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    {
+        // Perform the same hash calculation as cmake/CalculateEnginePathId.cmake
+        /////
+
+        // Prepare the engine path the same way as cmake/CalculateEnginePathId.cmake
+        AZStd::string enginePath = AZ::IO::FixedMaxPath(engineRoot).StringAsPosix();
+        enginePath += '/';
+        AZStd::to_lower(enginePath.begin(), enginePath.end());
+
+        // Perform a SHA1 hash on the prepared engine path
+        AZ::Sha1 hasher;
+        AZ::u32 digest[5];
+        hasher.ProcessBytes(AZStd::as_bytes(AZStd::span(enginePath)));
+        hasher.GetDigest(digest);
+
+        // Construct the path to where the python venv based on the engine path should be located
+        AZ::IO::FixedMaxPath libPath = thirdPartyRoot;
+        // The ID is based on the first 32 bits of the digest, and formatted to at least 8-character wide hexadecimal representation
+        libPath /= AZ::IO::FixedMaxPathString::format("venv/%08x", digest[0]);
+        libPath = libPath.LexicallyNormal();
+        return libPath;
+    }
+
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    {
+        AZ::IO::FixedMaxPath pythonVenvConfig = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot) / "pyvenv.cfg";
+        AZ::IO::SystemFileStream systemFileStream;
+        if (!systemFileStream.Open(pythonVenvConfig.c_str(), AZ::IO::OpenMode::ModeRead))
+        {
+            AZ_Error("python", false, "Missing python venv file at %s. Make sure to run python/get_python.", pythonVenvConfig.c_str());
+            return "";
+        }
+
+        AZ::Settings::ConfigParserSettings parserSettings;
+        AZ::IO::FixedMaxPathString pythonHome;
+        parserSettings.m_parseConfigEntryFunc = [&pythonHome](const AZ::Settings::ConfigParserSettings::ConfigEntry& configEntry)
+        {
+            if (AZ::StringFunc::Equal(configEntry.m_keyValuePair.m_key, "home"))
+            {
+                pythonHome = configEntry.m_keyValuePair.m_value;
+            }
+            return true;
+        };
+        const auto parseOutcome = AZ::Settings::ParseConfigFile(systemFileStream, parserSettings);
+        AZ_Error("python", parseOutcome, "Python venv file at %s missing home key. Make sure to run python/get_python.", pythonVenvConfig.c_str());
+
+        return AZ::IO::FixedMaxPath(pythonHome);
+    }
+
+    void PythonLoader::ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor resultPathCallback)
+    {
+        // Get the python venv path
+        AZ::IO::FixedMaxPath pythonVenvSitePackages =
+            AZ::IO::FixedMaxPath(PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot)) / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
+
+        // Always add the site-packages folder from the virtual environment into the path list
+        resultPathCallback(pythonVenvSitePackages.LexicallyNormal());
+
+        // pybind11 does not resolve any .egg-link files, so any packages that there pip-installed into the venv as egg-links
+        // are not getting resolved. We will do this manually by opening the egg-links in the venv site-packages path and injecting
+        // the non local paths as well
+        AZ::IO::LocalFileIO localFileSystem;
+        localFileSystem.FindFiles(
+            pythonVenvSitePackages.c_str(),
+            "*.egg-link",
+            [&resultPathCallback](const char* filePath) -> bool
+            {
+                auto readFileResult = AZ::Utils::ReadFile(filePath);
+                if (readFileResult)
+                {
+                    auto eggLinkContent = readFileResult.GetValue();
+
+                    AZStd::vector<AZStd::string> eggLinkLines;
+                    AZStd::string delim = "\r\n";
+                    AZStd::tokenize(eggLinkContent, delim, eggLinkLines);
+                    for (auto eggLinkLine : eggLinkLines)
+                    {
+                        if (eggLinkLine.compare(".") != 0)
+                        {
+                            resultPathCallback(AZ::IO::PathView(eggLinkLine));
+                        }
+                    }
+                }
+                return true;
+            }
+        );
+    }
+}

Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.h

@@ -8,6 +8,12 @@
 
 #pragma once
 
+#include <AzCore/IO/Path/Path.h>
+#include <AzCore/Module/DynamicModuleHandle.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/smart_ptr/unique_ptr.h>
+#include <AzCore/std/string/string.h>
+#include <AzCore/std/string/string_view.h>
 namespace AzToolsFramework::EmbeddedPython
 {
     // When using embedded Python, some platforms need to explicitly load the python library.
@@ -18,8 +24,39 @@ namespace AzToolsFramework::EmbeddedPython
         PythonLoader();
         ~PythonLoader();
 
+        //! Calculate the python home (PYTHONHOME) based on the engine root
+        //! @param engineRoot The path to the engine root to locate the python home
+        //! @return The path of the python home path
+        static AZ::IO::FixedMaxPath GetPythonHomePath(AZ::IO::PathView engineRoot);
+
+        //! Collect the paths from all the egg-link files found in the python home
+        //! paths used by the engine
+        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
+        //! @param engineRoot The path to the engine root to locate the python home
+        //! @param eggLinkPathVisitor The callback visitor function to receive the egg-link paths that are discovered
+        using EggLinkPathVisitor = AZStd::function<void(AZ::IO::PathView)>;
+        static void ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor eggLinkPathVisitor);
+
+        //! Get the default 3rd Party folder path.
+        //! @return The path of the 3rd Party root path
+        static AZ::IO::FixedMaxPath GetDefault3rdPartyPath(bool createOnDemand);
+
+        //! Calculate the path to the engine's python virtual environment used for
+        //! python home (PYTHONHOME) based on the engine root
+        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
+        //! @param engineRoot The path to the engine root to locate the python venv path
+        //! @return The path of the python venv path
+        static AZ::IO::FixedMaxPath GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+
+        //! Calculate the path to the where the python executable resides in. Note that this
+        //! is not always the same path as the python home path
+        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
+        //! @param engineRoot The path to the engine root to
+        //! locate the python executable path
+        //! @return The path of the python venv path
+        static AZ::IO::FixedMaxPath GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+
     private:
-        [[maybe_unused]] void* m_embeddedLibPythonHandle{ nullptr };
+        AZStd::unique_ptr<AZ::DynamicModuleHandle> m_embeddedLibPythonModuleHandle;
     };
-
 } // namespace AzToolsFramework::EmbeddedPython

Code/Framework/AzToolsFramework/AzToolsFramework/aztoolsframework_files.cmake

@@ -90,6 +90,7 @@ set(FILES
     API/EntityCompositionNotificationBus.h
     API/EditorViewportIconDisplayInterface.h
     API/PythonLoader.h
+    API/PythonLoader.cpp
     API/ViewPaneOptions.h
     API/ViewportEditorModeTrackerInterface.h
     Application/Ticker.h

Code/Framework/AzToolsFramework/CMakeLists.txt

@@ -32,6 +32,7 @@ ly_add_target(
     COMPILE_DEFINITIONS
         PRIVATE
             $<$<CONFIG:debug>:ENABLE_UNDOCACHE_CONSISTENCY_CHECKS>
+            O3DE_PYTHON_SITE_PACKAGE_SUBPATH="${LY_PYTHON_VENV_SITE_PACKAGES}"
     BUILD_DEPENDENCIES
         PRIVATE
             3rdParty::SQLite
@@ -80,6 +81,9 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
         INCLUDE_DIRECTORIES
             PRIVATE
                 Tests
+        COMPILE_DEFINITIONS
+            PRIVATE
+                O3DE_PYTHON_SITE_PACKAGE_SUBPATH="${LY_PYTHON_VENV_SITE_PACKAGES}"
         BUILD_DEPENDENCIES
             PUBLIC
                 AZ::AzTestShared

Code/Framework/AzToolsFramework/Platform/Common/Default/AzToolsFramework/API/PythonLoader_Default.cpp

@@ -1,20 +0,0 @@
-/*
-* Copyright (c) Contributors to the Open 3D Engine Project.
-* For complete copyright and license terms please see the LICENSE at the root of this distribution.
-*
-* SPDX-License-Identifier: Apache-2.0 OR MIT
-*
-*/
-
-#include <AzToolsFramework/API/PythonLoader.h>
-
-namespace AzToolsFramework::EmbeddedPython
-{
-    PythonLoader::PythonLoader()
-    {
-    }
-
-    PythonLoader::~PythonLoader()
-    {
-    }
-}

Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework/API/PythonLoader_Linux.cpp

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzToolsFramework/API/PythonLoader.h>
-#include <AzCore/Debug/Trace.h>
-#include <AzCore/IO/Path/Path.h>
-#include <dlfcn.h>
-
-namespace AzToolsFramework::EmbeddedPython
-{
-
-
-    PythonLoader::PythonLoader()
-    {
-        // PYTHON_SHARED_LIBRARY_PATH must be defined in the build scripts and referencing the path to the python shared library
-        #if !defined(PYTHON_SHARED_LIBRARY_PATH)
-        #error "PYTHON_SHARED_LIBRARY_PATH is not defined"
-        #endif
-        AZ::IO::FixedMaxPath libPythonName = AZ::IO::PathView(PYTHON_SHARED_LIBRARY_PATH).Filename();
-        m_embeddedLibPythonHandle = dlopen(libPythonName.c_str(), RTLD_NOW | RTLD_GLOBAL);
-        if (m_embeddedLibPythonHandle == nullptr)
-        {
-            [[maybe_unused]] const char* err = dlerror();
-            AZ_Error("PythonLoader", false, "Failed to load %s with error: %s\n", libPythonName.c_str(), err ? err : "Unknown Error");
-        }
-    }
-
-    PythonLoader::~PythonLoader()
-    {
-        if (m_embeddedLibPythonHandle)
-        {
-            dlclose(m_embeddedLibPythonHandle);
-        }
-    }
-    
-} // namespace AzToolsFramework::EmbeddedPython

Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Linux.h

@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      true
+#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      true

Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include <AzToolsFramework_Traits_Linux.h>

Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake

@@ -7,6 +7,7 @@
 #
 
 set(FILES
-    AzToolsFramework/API/PythonLoader_Linux.cpp
+    AzToolsFramework_Traits_Linux.h
+    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Linux.cpp
 )

Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Mac.h

@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      false
+#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      true

Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include <AzToolsFramework_Traits_Mac.h>

Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake

@@ -7,6 +7,7 @@
 #
 
 set(FILES
-    ../Common/Default/AzToolsFramework/API/PythonLoader_Default.cpp
+    AzToolsFramework_Traits_Mac.h
+    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Mac.cpp
 )

Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include <AzToolsFramework_Traits_Windows.h>

Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Windows.h

@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      false
+#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      false

Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake

@@ -7,6 +7,7 @@
 #
 
 set(FILES
-    ../Common/Default/AzToolsFramework/API/PythonLoader_Default.cpp
+    AzToolsFramework_Traits_Windows.h
+    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Windows.cpp
 )

Code/Framework/AzToolsFramework/Tests/PythonLoaderTests.cpp

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#include <AzCore/Component/Entity.h>
+#include <AzCore/RTTI/BehaviorContext.h>
+#include <AzCore/Serialization/SerializeContext.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/sort.h>
+#include <AzTest/AzTest.h>
+#include <AzTest/Utils.h>
+#include <AzCore/UnitTest/TestTypes.h>
+#include <AzToolsFramework/API/PythonLoader.h>
+
+namespace UnitTest
+{
+    class AzToolsFrameworkPythonLoaderFixture
+        : public LeakDetectionFixture
+    {
+    protected:
+        AZ::Test::ScopedAutoTempDirectory m_tempDirectory;
+
+        static constexpr AZStd::string_view s_testEnginePath = "O3de_path";
+        static constexpr const char* s_testEnginePathHashId = "1af80774";
+        static constexpr const char* s_test3rdPartySubPath = ".o3de/3rdParty";
+    };
+
+    TEST_F(AzToolsFrameworkPythonLoaderFixture, TestGetPythonVenvPath_Valid)
+    {
+        auto test3rdPartyPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
+        auto testVenvPath = test3rdPartyPath / "venv";
+
+        AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
+
+        AZ::IO::SystemFile::CreateDir(test3rdPartyPath.String().c_str());
+        //AZ::IO::FileIOBase::GetInstance()->CreatePath(test3rdPartyPath.String().c_str());
+
+        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonVenvPath(test3rdPartyPath, engineRoot);
+
+        AZ::IO::FixedMaxPath expectedPath = testVenvPath / s_testEnginePathHashId;
+
+        EXPECT_TRUE(result == expectedPath);
+    }
+
+    TEST_F(AzToolsFrameworkPythonLoaderFixture, TestGetPythonVenvExecutablePath_Valid)
+    {
+        // Prepare the test venv pyvenv.cfg file in the expected location
+        AZ::IO::FixedMaxPath tempVenvRelativePath = AZ::IO::FixedMaxPath(s_test3rdPartySubPath) / "venv/" / s_testEnginePathHashId;
+        AZ::IO::FixedMaxPath tempVenvFullPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / tempVenvRelativePath;
+        AZ::IO::SystemFile::CreateDir(tempVenvFullPath.String().c_str());
+        AZ::IO::FixedMaxPath tempPyConfigFile = tempVenvRelativePath / "pyvenv.cfg";
+        AZStd::string testPython3rdPartyPath = "/home/user/python/";
+        AZStd::string testPyConfigFileContent = AZStd::string::format("home = %s\n"
+                                                                      "include-system-site-packages = false\n"
+                                                                      "version = 3.10.13",
+                                                                      testPython3rdPartyPath.c_str());
+        AZ::Test::CreateTestFile(m_tempDirectory, tempPyConfigFile, testPyConfigFileContent);
+
+
+        // Test the method
+        AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
+        AZ::IO::FixedMaxPath test3rdPartyRoot = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
+
+        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonExecutablePath(test3rdPartyRoot, engineRoot);
+        AZ::IO::FixedMaxPath expectedPath{ testPython3rdPartyPath };
+
+        EXPECT_TRUE(result == expectedPath);
+    }
+
+
+    TEST_F(AzToolsFrameworkPythonLoaderFixture, TestReadPythonEggLinkPaths_Valid)
+    {
+        // Prepare the test folder and create dummy egg-link files
+        AZ::IO::FixedMaxPath testRelativeSiteLibsPath = AZ::IO::FixedMaxPath(s_test3rdPartySubPath) / "venv" / s_testEnginePathHashId / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
+        AZ::IO::FixedMaxPath testFullSiteLIbsPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / testRelativeSiteLibsPath;
+        AZ::IO::SystemFile::CreateDir(testFullSiteLIbsPath.String().c_str());
+
+        AZStd::vector<AZStd::string> expectedResults;
+        expectedResults.emplace_back(testFullSiteLIbsPath.LexicallyNormal().Native());
+
+        static constexpr auto testEggLinkPaths = AZStd::to_array<const char*>({ "/lib/path/one", "/lib/path/two", "/lib/path/three" });
+        int index = 0;
+        for (const char* testEggLinkPath : testEggLinkPaths)
+        {
+            ++index;
+            AZStd::string testEggFileName = AZStd::string::format("test-%d.egg-link", index);
+            const char* lineBreak = ((index % 2) == 0) ? "\n" : "\r\n";
+            AZStd::string testEggFileContent = AZStd::string::format("%s%s.", testEggLinkPath, lineBreak);
+            expectedResults.emplace_back(AZStd::string(testEggLinkPath));
+
+            AZ::IO::FixedMaxPath testEggLinkNamePath = testRelativeSiteLibsPath / testEggFileName;
+            AZ::Test::CreateTestFile(m_tempDirectory, testEggLinkNamePath, testEggFileContent);
+        }
+
+        // Test the method
+        AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
+        AZ::IO::FixedMaxPath test3rdPartyRoot = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
+        AZStd::vector<AZStd::string> resultEggLinkPaths;
+        AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
+            test3rdPartyRoot,
+            engineRoot,
+            [&resultEggLinkPaths](AZ::IO::PathView path)
+            {
+                resultEggLinkPaths.emplace_back(path.Native());
+            } );
+
+        // Sort the expected and actual lists
+        AZStd::sort(expectedResults.begin(), expectedResults.end());
+        AZStd::sort(resultEggLinkPaths.begin(), resultEggLinkPaths.end());
+
+        EXPECT_EQ(expectedResults, resultEggLinkPaths);
+    }
+
+} // namespace UnitTest

Code/Framework/AzToolsFramework/Tests/aztoolsframeworktests_files.cmake

@@ -169,6 +169,7 @@ set(FILES
     PropertyIntSpinCtrlTests.cpp
     PropertyTreeEditorTests.cpp
     PythonBindingTests.cpp
+    PythonLoaderTests.cpp
     QtWidgetLimitsTests.cpp
     Script/LuaEditorSystemComponentTests.cpp
     Script/ScriptComponentTests.cpp

Code/Tools/ProjectManager/CMakeLists.txt

@@ -67,6 +67,7 @@ ly_add_target(
             AZ::AzCore
             AZ::AzFramework
             AZ::AzQtComponents
+            AZ::AzToolsFramework
             AZ::ProjectManager.Static
 )
 
@@ -94,6 +95,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
                 AZ::AzFrameworkTestShared
                 AZ::AzQtComponents
                 AZ::ProjectManager.Static
+                AZ::AzToolsFramework
     )
 
     ly_add_googletest(

Code/Tools/ProjectManager/Platform/Linux/PAL_linux.cmake

@@ -9,4 +9,4 @@
 set(LY_COMPILE_DEFINITIONS 
     PRIVATE 
         PY_VERSION_MAJOR_MINOR="${LY_PYTHON_VERSION_MAJOR_MINOR}"
-)
+)

Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake

@@ -7,7 +7,6 @@
 #
 
 set(FILES
-    Python_linux.cpp
     ProjectBuilderWorker_linux.cpp
     ProjectUtils_linux.cpp
     ProjectManagerDefs_linux.cpp

Code/Tools/ProjectManager/Platform/Linux/ProjectManager_Traits_Linux.h

@@ -10,3 +10,4 @@
 
 #define AZ_TRAIT_PROJECT_MANAGER_CUSTOM_TITLEBAR false
 #define AZ_TRAIT_PROJECT_MANAGER_CREATE_DESKTOP_SHORTCUT false
+#define AZ_TRAIT_PROJECT_MANAGER_PYTHON_EXECUTABLE_SUBPATH "python/python.sh"

Code/Tools/ProjectManager/Platform/Linux/Python_linux.cpp

@@ -1,53 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzCore/PlatformDef.h>
-#include <AzCore/std/containers/unordered_set.h>
-#include <AzCore/IO/SystemFile.h>
-#include <AzCore/IO/Path/Path.h>
-#include <AzFramework/StringFunc/StringFunc.h>
-
-namespace Platform
-{
-    extern bool InsertPythonLibraryPath(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot, const char* subPath);
-
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot)
-    {
-        bool succeeded = true;
-
-        // PY_VERSION_MAJOR_MINOR must be defined through the build scripts based on the current python package (see cmake/LYPython.cmake)
-        #if !defined(PY_VERSION_MAJOR_MINOR)
-        #error "PY_VERSION_MAJOR_MINOR is not defined"
-        #endif
-
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR "/lib-dynload");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR );
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR "/site-packages");
-
-        return succeeded;
-    }
-
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format("python/runtime/%s/python", pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-
-    AZStd::string GetPythonExecutablePath(const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString("python/python.sh");
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-}

Code/Tools/ProjectManager/Platform/Mac/PAL_mac.cmake

@@ -10,4 +10,3 @@ set(LY_COMPILE_DEFINITIONS
     PRIVATE 
         PY_VERSION_MAJOR_MINOR="${LY_PYTHON_VERSION_MAJOR_MINOR}"
 )
-

Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake

@@ -7,7 +7,6 @@
 #
 
 set(FILES
-    Python_mac.cpp
     ProjectBuilderWorker_mac.cpp
     ProjectUtils_mac.cpp
     ProjectManagerDefs_mac.cpp

Code/Tools/ProjectManager/Platform/Mac/ProjectManager_Traits_Mac.h

@@ -10,3 +10,4 @@
 
 #define AZ_TRAIT_PROJECT_MANAGER_CUSTOM_TITLEBAR false
 #define AZ_TRAIT_PROJECT_MANAGER_CREATE_DESKTOP_SHORTCUT false
+#define AZ_TRAIT_PROJECT_MANAGER_PYTHON_EXECUTABLE_SUBPATH "python/python.sh"

Code/Tools/ProjectManager/Platform/Mac/Python_mac.cpp

@@ -1,53 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzCore/PlatformDef.h>
-#include <AzCore/std/containers/unordered_set.h>
-#include <AzCore/IO/SystemFile.h>
-#include <AzCore/IO/Path/Path.h>
-#include <AzFramework/StringFunc/StringFunc.h>
-
-namespace Platform
-{
-    extern bool InsertPythonLibraryPath(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot, const char* subPath);
-
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot)
-    {
-        bool succeeded = true;
-        
-        // PY_VERSION_MAJOR_MINOR must be defined through the build scripts based on the current python package (see cmake/LYPython.cmake)
-        #if !defined(PY_VERSION_MAJOR_MINOR)
-        #error "PY_VERSION_MAJOR_MINOR is not defined"
-        #endif
-
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR "/lib-dynload");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR);
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR "/site-packages");
-
-        return succeeded;
-    }
-
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format("python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR, pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-
-    AZStd::string GetPythonExecutablePath(const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString("python/python.sh");
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-}

Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake

@@ -7,7 +7,6 @@
 #
 
 set(FILES
-    Python_windows.cpp
     ProjectBuilderWorker_windows.cpp
     ProjectUtils_windows.cpp
     ProjectManagerDefs_windows.cpp

Code/Tools/ProjectManager/Platform/Windows/ProjectManager_Traits_Windows.h

@@ -10,3 +10,4 @@
 
 #define AZ_TRAIT_PROJECT_MANAGER_CUSTOM_TITLEBAR true
 #define AZ_TRAIT_PROJECT_MANAGER_CREATE_DESKTOP_SHORTCUT true
+#define AZ_TRAIT_PROJECT_MANAGER_PYTHON_EXECUTABLE_SUBPATH "python\\python.cmd"

Code/Tools/ProjectManager/Source/Application.cpp

@@ -76,45 +76,31 @@ namespace O3DE::ProjectManager
         // unit tests may provide custom python bindings 
         m_pythonBindings = pythonBindings ? AZStd::move(pythonBindings) : AZStd::make_unique<PythonBindings>(GetEngineRoot());
 
+
+        // If the first attempt of starting python failed, then attempt to bootstrap python by 
+        // calling the get python script
         if (!m_pythonBindings->PythonStarted())
         {
-            if (!interactive)
-            {
-                return false;
-            }
-
-            int result = QMessageBox::warning(nullptr, QObject::tr("Failed to start Python"),
-                QObject::tr("This tool requires an O3DE engine with a Python runtime, "
-                            "but either Python is missing or mis-configured.<br><br>Press 'OK' to "
-                            "run the %1 script automatically, or 'Cancel' "
-                            " if you want to manually resolve the issue by renaming your "
-                            " python/runtime folder and running %1 yourself.")
-                            .arg(GetPythonScriptPath),
-                QMessageBox::Cancel, QMessageBox::Ok);
-            if (result == QMessageBox::Ok)
+            auto getPythonResult = ProjectUtils::RunGetPythonScript(GetEngineRoot());
+            if (getPythonResult.IsSuccess())
             {
-                auto getPythonResult = ProjectUtils::RunGetPythonScript(GetEngineRoot());
-                if (!getPythonResult.IsSuccess())
-                {
-                    QMessageBox::critical(
-                        nullptr, QObject::tr("Failed to run %1 script").arg(GetPythonScriptPath),
-                        QObject::tr("The %1 script failed, was canceled, or could not be run.  "
-                                    "Please rename your python/runtime folder and then run "
-                                    "<pre>%1</pre>").arg(GetPythonScriptPath));
-                }
-                else if (!m_pythonBindings->StartPython())
-                {
-                    QMessageBox::critical(
-                        nullptr, QObject::tr("Failed to start Python"),
-                        QObject::tr("Failed to start Python after running %1")
-                                    .arg(GetPythonScriptPath));
-                }
+                // If the bootstrap for python was successful, then attempt to start python again
+                m_pythonBindings->StartPython();
             }
+        }
 
-            if (!m_pythonBindings->PythonStarted())
+        if (!m_pythonBindings->PythonStarted())
+        {
+            if (interactive)
             {
-                return false;
+                QMessageBox::critical(nullptr, QObject::tr("Failed to start Python"),
+                QObject::tr("This tool requires an O3DE engine with a Python runtime, "
+                            "but was unable to automatically install O3DE's built-in Python."
+                            "You can troubleshoot this issue by trying to manually install O3DE's built-in "
+                            "Python by running the '%1' script.")
+                            .arg(GetPythonScriptPath));
             }
+            return false;
         }
 
         m_settings = AZStd::make_unique<Settings>();

Code/Tools/ProjectManager/Source/Application.h

@@ -9,6 +9,7 @@
 
 #if !defined(Q_MOC_RUN)
 #include <AzFramework/Application/Application.h>
+#include <AzToolsFramework/API/PythonLoader.h>
 #include <QCoreApplication>
 #include <PythonBindings.h>
 #include <Settings.h>
@@ -24,6 +25,7 @@ namespace O3DE::ProjectManager
 {
     class Application
         : public AzFramework::Application
+        , public AzToolsFramework::EmbeddedPython::PythonLoader
     {
     public:
         AZ_CLASS_ALLOCATOR(Application, AZ::SystemAllocator)

Code/Tools/ProjectManager/Source/ProjectUtils.cpp

@@ -8,6 +8,7 @@
 
 #include <ProjectUtils.h>
 #include <ProjectManagerDefs.h>
+#include <ProjectManager_Traits_Platform.h>
 #include <PythonBindingsInterface.h>
 #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzCore/IO/Path/Path.h>
@@ -717,6 +718,15 @@ namespace O3DE::ProjectManager
             return AZ::Success(QString(projectBuildPath.c_str()));
         }
 
+        QString GetPythonExecutablePath(const QString& enginePath)
+        {
+            // append lib path to Python paths
+            AZ::IO::FixedMaxPath libPath = enginePath.toUtf8().constData();
+            libPath /= AZ::IO::FixedMaxPathString(AZ_TRAIT_PROJECT_MANAGER_PYTHON_EXECUTABLE_SUBPATH);
+            libPath = libPath.LexicallyNormal();
+            return QString(libPath.String().c_str());
+        }
+
         QString GetDefaultProjectPath()
         {
             QString defaultPath = QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation);

Code/Tools/ProjectManager/Source/ProjectsScreen.cpp

@@ -641,7 +641,7 @@ namespace O3DE::ProjectManager
             projectName = getProjectResult.GetValue().m_displayName;
         }
 
-        const QString pythonPath = GetPythonExecutablePath(engineInfo.m_path);
+        const QString pythonPath = ProjectUtils::GetPythonExecutablePath(engineInfo.m_path);
         const QString apgPath = QString("%1/Code/Tools/Android/ProjectGenerator/main.py").arg(engineInfo.m_path);
 
 

Code/Tools/ProjectManager/Source/PythonBindings.cpp

@@ -9,6 +9,7 @@
 #include <PythonBindings.h>
 
 #include <ProjectManagerDefs.h>
+#include <osdefs.h> // for DELIM
 
 // Qt defines slots, which interferes with the use here.
 #pragma push_macro("slots")
@@ -26,6 +27,8 @@
 #include <AzCore/std/numeric.h>
 #include <AzCore/StringFunc/StringFunc.h>
 #include <AzCore/std/sort.h>
+#include <AzToolsFramework/API/PythonLoader.h>
+
 
 #include <QDir>
 
@@ -47,13 +50,6 @@ namespace Platform
         AZ_Warning("python", false, "Python library path should exist. path:%s", libPath.c_str());
         return false;
     }
-
-    // Implemented in each different platform's PAL implementation files, as it differs per platform.
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot);
-
-    // Per platform information.
-    AZStd::string GetPythonExecutablePath(const char* engineRoot);
-
 } // namespace Platform
 
 
@@ -239,11 +235,6 @@ namespace RedirectOutput
 
 namespace O3DE::ProjectManager
 {
-    QString GetPythonExecutablePath(const QString& enginePath)
-    {
-        return QString(Platform::GetPythonExecutablePath(enginePath.toUtf8().constData()).c_str());
-    }
-
     PythonBindings::PythonBindings(const AZ::IO::PathView& enginePath)
         : m_enginePath(enginePath)
     {
@@ -293,7 +284,7 @@ namespace O3DE::ProjectManager
         m_pythonStarted = false;
 
         // set PYTHON_HOME
-        AZStd::string pyBasePath = Platform::GetPythonHomePath(PY_PACKAGE, m_enginePath.c_str());
+        AZStd::string pyBasePath = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonHomePath(m_enginePath).StringAsPosix();
         if (!AZ::IO::SystemFile::Exists(pyBasePath.c_str()))
         {
             AZ_Error("python", false, "Python home path does not exist: %s", pyBasePath.c_str());
@@ -323,6 +314,24 @@ namespace O3DE::ProjectManager
 
             RedirectOutput::Intialize(PyImport_ImportModule("azlmbr_redirect"), &PythonBindings::OnStdOut, &PythonBindings::OnStdError);
 
+            // Add custom site packages after initializing the interpreter above.  Calling Py_SetPath before initialization
+            // alters the behavior of the initializer to not compute default search paths. See
+            // https://docs.python.org/3/c-api/init.html#c.Py_SetPath
+
+            AZ::IO::FixedMaxPath thirdPartyFolder = AzToolsFramework::EmbeddedPython::PythonLoader::GetDefault3rdPartyPath(false);
+
+            AZStd::vector<AZ::IO::Path> extendedPaths;
+            AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
+                thirdPartyFolder, m_enginePath.c_str(), [&extendedPaths](AZ::IO::PathView path)
+                {
+                    extendedPaths.emplace_back(path);
+                });
+
+            if (!extendedPaths.empty())
+            {
+                ExtendSysPath(extendedPaths);
+            }
+
             // Acquire GIL before calling Python code
             AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
             pybind11::gil_scoped_acquire acquire;
@@ -2067,4 +2076,31 @@ namespace O3DE::ProjectManager
     {
         m_pythonErrorStrings.push_back(errorString);
     }
+
+    bool PythonBindings::ExtendSysPath(const AZStd::vector<AZ::IO::Path>& extendPaths)
+    {
+        AZStd::unordered_set<AZ::IO::Path> oldPathSet;
+        auto SplitPath = [&oldPathSet](AZStd::string_view pathPart)
+        {
+            oldPathSet.emplace(AZ::IO::FixedMaxPath(pathPart));
+        };
+        AZ::StringFunc::TokenizeVisitor(Py_EncodeLocale(Py_GetPath(), nullptr), SplitPath, DELIM);
+        bool appended{ false };
+        AZStd::string pathAppend{ "import sys\n" };
+        for (const auto& thisStr : extendPaths)
+        {
+            if (!oldPathSet.contains(thisStr.c_str()))
+            {
+                pathAppend.append(AZStd::string::format("sys.path.append(r'%s')\n", thisStr.c_str()));
+                appended = true;
+            }
+        }
+        if (appended)
+        {
+            PyRun_SimpleString(pathAppend.c_str());
+            return true;
+        }
+        return false;
+    }
+
 } // namespace O3DE::ProjectManager

Code/Tools/ProjectManager/Source/PythonBindings.h

@@ -21,8 +21,6 @@
 
 namespace O3DE::ProjectManager
 {
-    QString GetPythonExecutablePath(const QString& enginePath);
-
     class PythonBindings
         : public PythonBindingsInterface::Registrar
     {
@@ -120,7 +118,8 @@ namespace O3DE::ProjectManager
         AZ::Outcome<void, AZStd::string> GemRegistration(const QString& gemPath, const QString& projectPath, bool remove = false);
         bool StopPython();
         IPythonBindings::ErrorPair GetErrorPair();
-
+        bool ExtendSysPath(const AZStd::vector<AZ::IO::Path>& extendPaths);
+    
         bool m_pythonStarted = false;
 
         AZ::IO::FixedMaxPath m_enginePath;

Gems/AWSClientAuth/cdk/README.md

@@ -21,7 +21,7 @@ The `cdk.json` file tells the CDK Toolkit how to execute your app.
 
 This project is set up like a standard Python project. You can use either a [python virtual environment](https://docs.aws.amazon.com/cdk/v2/guide/work-with-cdk-python.html) or use python interpreter from O3DE to setup python dependencies ie:
 ```
-set PATH="..\..\..\python\runtime\python-3.10.5-rev1-windows\python\";%PATH%
+set PATH="..\..\..\python";%PATH%
 ```
 
 Once the python and pip are set up, you can install the required dependencies. To setup with O3DE's python interpreter use:

Gems/AWSClientAuth/cdkv1/README.md

@@ -18,7 +18,7 @@ Run from cdk directory within gem.
 
 Optional set up python path to LY Python
 ```
-set PATH="..\..\..\python\runtime\python-3.10.5-rev1-windows\python\";%PATH%
+set PATH="..\..\..\python\";%PATH%
 ```
 
 This project is set up like a standard Python project.  Use python interpreter from Open3d to setup python dependencies

Gems/AWSMetrics/cdk/README.md

@@ -15,7 +15,7 @@ process also creates a virtualenv within this project, stored under the .env
 directory.  To create the virtualenv it assumes that there is a `python3`
 (or `python` for Windows) executable in your path with access to the `venv`
 package. If for any reason the automatic creation of the virtualenv fails,
-you can create the virtualenv manually. Please note that Python version 3.10.5 or higher is required to deploy this CDK application.
+you can create the virtualenv manually. Please note that Python version 3.10.13 or higher is required to deploy this CDK application.
 
 To manually create a virtualenv on macOS and Linux:
 

Gems/Atom/RPI/Tools/README.txt

@@ -17,7 +17,7 @@ developed by the Atom team. The project contains the following tools:
 REQUIREMENTS
 ------------
 
- * Python 3.10.5 (64-bit)
+ * Python 3.10.13 (64-bit)
 
 It is recommended that you completely remove any other versions of Python
 installed on your system.

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Atom/Scripts/Python/DCC_Materials/maya_materials_export.py

@@ -243,7 +243,7 @@ class MayaToLumberyard(QtWidgets.QWidget):
         # Setup Lumberyard Material File Values
         self.map_materials()
         print ('MaterialDefinitions:'.format(self.material_definitions))
-        print json.dumps(self.material_definitions, sort_keys=True, indent=4)
+        print(json.dumps(self.material_definitions, sort_keys=True, indent=4))
 
         # Update UI Layout
         self.set_material_view()

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Lumberyard/Scripts/set_menu.py

@@ -57,7 +57,7 @@ if settings.DCCSI_DEV_MODE:
 try:
     azlmbr.qt.QtForPythonRequestBus(azlmbr.bus.Broadcast, 'IsActive')
     params = azlmbr.qt.QtForPythonRequestBus(azlmbr.bus.Broadcast, 'GetQtBootstrapParameters')
-    params is not None and params.mainWindowId is not 0
+    params is not None and params.mainWindowId != 0
     from PySide2 import QtWidgets
 except Exception as e:
     _LOGGER.error(f'Pyside not available, exception: {e}')

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Maya/Scripts/Python/kitbash_converter/main.py

@@ -1022,14 +1022,14 @@ class AttributeListing(QtWidgets.QWidget):
         if isinstance(value, list):
             converted_list = []
             for item in value:
-                item_value = 0 if item is 0 else float(item)
+                item_value = 0 if item == 0 else float(item)
                 converted_list.append(item)
             return converted_list
         elif isinstance(value, str):
             if value.lower() in ['true', 'false']:
                 return bool(value.capitalize())
             elif value.isnumeric():
-                return 0 if value is 0 else float(value)
+                return 0 if value == 0 else float(value)
             else:
                 pattern = r'\[[^\]]*\]'
                 test_brackets = re.sub(pattern, '', value)

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Maya/Scripts/Python/maya_dcc_materials/maya_materials_export.py

@@ -248,7 +248,7 @@ class MayaToLumberyard(QtWidgets.QWidget):
         # Setup Lumberyard Material File Values
         self.map_materials()
         print ('MaterialDefinitions:'.format(self.material_definitions))
-        print json.dumps(self.material_definitions, sort_keys=True, indent=4)
+        print(json.dumps(self.material_definitions, sort_keys=True, indent=4))
 
         # Update UI Layout
         self.set_material_view()

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/__init__.py

@@ -19,7 +19,7 @@ from azpy.constants import ENVAR_DCCSI_DEV_MODE
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.Substance.builder'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/atom_material.py

@@ -33,7 +33,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.atom_material'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/sbs_to_sbsar.py

@@ -51,7 +51,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.sbs_to_sbsar'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/sbsar_info.py

@@ -51,7 +51,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.sbsar_info'
 
 import azpy
@@ -75,7 +75,7 @@ _TOOL_TAG = 'sdk.substance.builder.sbsar_info'
 _TYPE_TAG = 'module'
 
 _MODULENAME = __name__
-if _MODULENAME is '__main__':
+if _MODULENAME == '__main__':
     _MODULENAME = _TOOL_TAG
 # -------------------------------------------------------------------------
 

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/sbsar_render.py

@@ -49,7 +49,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.sbsar_render'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/substance_tools.py

@@ -50,7 +50,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.substance_tools'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/SDK/Substance/builder/watchdog/__init__.py

@@ -57,7 +57,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'DCCsi.SDK.substance.builder.watchdog'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/Tools/DCC/Maya/readme.md

@@ -306,7 +306,7 @@ mayapy2 -m pip install -r C:\Depot\o3de-dev\Gems\AtomLyIntegration\TechnicalArt\
 
 ![image](https://user-images.githubusercontent.com/23222931/155037710-79bee421-1355-484b-8c96-f672157da40a.png)
 
-### Maya 2022 (Python 3.10.5)
+### Maya 2022 (Python 3.10.13)
 
 This also is not very different, you just need to modify some of the commands. Also because O3DE is also on a version of py3.10.x, we can re-use the requirements.txt file that is in the root of the DccScriptingInterface gem folder.
 

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/Tools/DCC/Substance/readme.md

@@ -82,7 +82,7 @@ Many DCC applications such as Substance Designer also come with a managed python
 
 - O3DE doesn't use a user or system installed python interpreter, nor do most DCC apps we are aware of.
 
-- The DCC apps python version may be different then O3DE e.g. python 3.10.5 vs 3.9.9.
+- The DCC apps python version may be different then O3DE e.g. python 3.10.13 vs 3.9.9.
 
 - O3DE and most DCC apps we are aware of don't make use of virtual environments, which is a common way to manage package dependencies for different versions of python.
 

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/Tools/IDE/Wing/.solutions/DCCsi_8x.wpr

@@ -88,7 +88,7 @@ debug.launch-configs = (2,
                  ['']),
          'name': 'O3DE_DEV',
          'pyexec': ('project',
-                    '${O3DE_DEV}\\python\\runtime\\python-3.10.5-rev1-windows\\python\\python.exe'),
+                    '${O3DE_DEV}\\python\\python.cmd'),
          'pypath': ('project',
                     []),
          'pyrunargs': ('project',

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/Tools/IDE/Wing/readme.md

@@ -127,7 +127,7 @@ The DCCsi helps with aspects such as, configuration and settings, launching DCC
 Many IDEs such as Wing allow you to configure one or more launch configurations, each configuration can be used to define an interpretter as well as other aspects such as a environment or additional commands.  With the DCCsi, we can use these to define each DCC applications python, so we can test code by running it in that iterpretter.  These are defined in a data-driven way by an EVAR in the launch environment. The ENVAR is set to a path string of the interpretter location:
 
 The project default is currently configured like this inside of wing:
-  - `${O3DE_DEV}` = '${O3DE_DEV}\python\runtime\python-3.10.5-rev1-windows\python\python.exe'
+  - `${O3DE_DEV}` = '${O3DE_DEV}\python\python.cmd'
 
 You can inspect the project default settings:
 
@@ -235,7 +235,7 @@ In the root of the DCCsi, one of the files will start O3DE python, you can start
 
 The actual interpreter that runs is somewhere like
 
-    `o3de\python\runtime\python-3.10.5-rev1-windows\python\python.exe`
+    `o3de\python\python.cmd`
 
 The folder for Wing developers is here:
 

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dcc/maya/helpers/__init__.py

@@ -23,7 +23,7 @@ _DCCSI_GDEBUG = env_bool.env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool.env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'azpy.dcc.maya.helpers'
 
 # set up module logging

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dcc/maya/helpers/undo_context.py

@@ -39,7 +39,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'azpy.dcc.maya.helpers.undo_context'
 
 _LOGGER = initialize_logger(_PACKAGENAME, default_log_level=int(20))

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dcc/maya/helpers/utils.py

@@ -36,7 +36,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'azpy.dcc.maya.helpers.utils'
 
 _LOGGER = initialize_logger(_PACKAGENAME, default_log_level=int(20))

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dcc/maya/toolbits/__init__.py

@@ -21,7 +21,7 @@ _DCCSI_GDEBUG = env_bool.env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool.env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'azpy.dcc.maya.toolbits'
 
 # set up module logging

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dcc/maya/toolbits/detach.py

@@ -44,7 +44,7 @@ _DCCSI_GDEBUG = env_bool(ENVAR_DCCSI_GDEBUG, False)
 _DCCSI_DEV_MODE = env_bool(ENVAR_DCCSI_DEV_MODE, False)
 
 _PACKAGENAME = __name__
-if _PACKAGENAME is '__main__':
+if _PACKAGENAME == '__main__':
     _PACKAGENAME = 'azpy.dcc.maya.toolbits.detatch'
 
 import azpy

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/dev/ide/examples/maya_command_script.py

@@ -52,7 +52,7 @@ size = random.uniform(0.5, 2.0)
 variation = random.uniform(1.5, 5.0)
 amount = random.randint(9, 21)
 
-def make_some_wonky_cubes(name=, size, variation, amount):
+def make_some_wonky_cubes(name, size, variation, amount):
     # remove previous
     obj_list = cmds.ls('{}*'.format(name))
     if len(obj_list) > 0:

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/azpy/shared/ui/qtextedit_stdout.py

@@ -45,7 +45,7 @@ _TOOL_TAG = 'azpy.shared.ui.pyside2_qtextedit_stdout'
 _TYPE_TAG = 'test'
 
 _MODULENAME = __name__
-if _MODULENAME is '__main__':
+if _MODULENAME == '__main__':
     _MODULENAME = _TOOL_TAG
 
 if _DCCSI_GDEBUG:

Gems/AtomLyIntegration/TechnicalArt/DccScriptingInterface/python.sh

@@ -16,20 +16,85 @@ while [[ -h "$SOURCE" ]]; do
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-if [[ "$OSTYPE" == *"darwin"* ]]; then
-    PYTHON=$DIR/../../../../runtime/python-3.10.5-rev2-darwin/Python.framework/Versions/3.10/bin/python3
-elif [[ "$OSTYPE" == "msys" ]]; then
-    PYTHON=$DIR/../../../../runtime/python-3.10.5-rev1-windows/python/python.exe
+
+# Locate and make sure cmake is in the path
+if [[ "$OSTYPE" = *"darwin"* ]];
+then
+    PAL=Mac
+    ARCH=
 else
-    PYTHON=$DIR/../../../../runtime/python-3.10.5-rev2-linux/python/bin/python
+    PAL=Linux
+    ARCH=$( uname -m )
+fi
+
+if ! [ -x "$(command -v cmake)" ]; then
+    if [ -z ${LY_CMAKE_PATH} ]; then
+        echo "ERROR: Could not find cmake on the PATH and LY_CMAKE_PATH is not defined, cannot continue."
+        echo "Please add cmake to your PATH, or define LY_CMAKE_PATH"
+        exit 1
+    fi
+
+    export PATH=$LY_CMAKE_PATH:$PATH
+    if ! [ -x "$(command -v cmake)" ]; then
+        echo "ERROR: Could not find cmake on the PATH or at the known location: $LY_CMAKE_PATH"
+        echo "Please add cmake to the environment PATH or place it at the above known location."
+        exit 1
+    fi
+fi
+
+# Calculate the engine ID
+CALC_PATH=$DIR/../../../../cmake/CalculateEnginePathId.cmake
+LY_ROOT_FOLDER=$DIR/../../../..
+ENGINE_ID=$(cmake -P $CALC_PATH $LY_ROOT_FOLDER)
+if [ $? -ne 0 ]
+then
+    echo "Unable to calculate engine ID"
+    exit 1
+fi
+
+if [ "$LY_3RDPARTY_PATH" == "" ]
+then
+    LY_3RDPARTY_PATH=$HOME/.o3de/3rdParty
 fi
 
-if [[ -e "$PYTHON" ]];
+# Set the expected location of the python venv for this engine and the locations of the critical scripts/executables 
+# needed to run python within the venv properly
+PYTHON_VENV=$LY_3RDPARTY_PATH/venv/$ENGINE_ID
+PYTHON_VENV_ACTIVATE=$PYTHON_VENV/bin/activate
+PYTHON_VENV_PYTHON=$PYTHON_VENV/bin/python
+if [ ! -f $PYTHON_VENV_PYTHON ]
 then
-    PYTHONNOUSERSITE=1 "$PYTHON" "$@"
-    exit $?
+    echo "Python has not been downloaded/configured yet."    
+    echo "Try running $LY_ROOT_FOLDER/python/get_python.sh first."
+    exit 1
 fi
 
-echo "Python not found in $DIR/../../../.."
-echo "Try running $DIR/../../../../get_python.sh first."
-exit 1
+# Determine the current package from where the current venv was initiated from
+PYTHON_VENV_HASH_FILE=$PYTHON_VENV/.hash
+if [ ! -f $PYTHON_VENV_HASH_FILE ]
+then
+    echo "Python has not been downloaded/configured yet."
+    echo "Try running $LY_ROOT_FOLDER/python/get_python.sh first."
+    exit 1
+fi
+PYTHON_VENV_HASH=$(cat $PYTHON_VENV_HASH_FILE)
+
+# Calculate the expected hash from the current python package
+CURRENT_PYTHON_PACKAGE_HASH=$(cmake -P $DIR/../../../../python/get_python_package_hash.cmake $DIR/.. $PAL $ARCH)
+
+if [ "$PYTHON_VENV_HASH" != "$CURRENT_PYTHON_PACKAGE_HASH" ]
+then
+    echo "Python has been updated since the last time the python command was invoked."
+    echo "Run $LY_ROOT_FOLDER/python/get_python.sh to update."
+    exit 1
+fi
+
+# The python in the venv environment is a symlink which will cause issues with loading the python shared
+# object that is relative to the original python lib shared library in the package.
+PYTHON_LIB_PATH=$(awk -F ' = ' '/home/ {print $2}' $LY_3RDPARTY_PATH/venv/$ENGINE_ID/pyvenv.cfg | sed 's/python\/bin/python\/lib/g')
+
+source $PYTHON_VENV_ACTIVATE
+
+PYTHONNOUSERSITE=1 LD_LIBRARY_PATH="$PYTHON_LIB_PATH:$LD_LIBRARY_PATH" "$PYTHON_VENV_PYTHON" "$@"
+exit $?
+

Gems/AudioEngineWwise/README.md

@@ -1,2 +1,2 @@
-The AudioEngineWwise Gem has been moved to the O3DE Extras repo (https://github.com/o3de/o3de-extras).
-You may also download the AudioEngineWwise Gem from the Project Manager or O3DE CLI.
+The AudioEngineWwise Gem has been moved to the O3DE Extras repo (https://github.com/o3de/o3de-extras).
+You may also download the AudioEngineWwise Gem from the Project Manager or O3DE CLI.

Gems/EditorPythonBindings/Code/CMakeLists.txt

@@ -10,25 +10,14 @@ if(NOT PAL_TRAIT_BUILD_HOST_TOOLS)
     return()
 endif()
 
-# This will set python_package_name to whatever the package 'Python' is associated with
-ly_get_package_association(Python python_package_name)
-if (NOT python_package_name)
-    set(python_package_name "python-no-package-assocation-found")
-    message(WARNING "Python was not found in the package assocation list.  Did someone call ly_associate_package(xxxxxxx Python) ?")
-endif()
-    
 ly_add_target(
     NAME ${gem_name}.Static STATIC
     NAMESPACE Gem
     FILES_CMAKE
         editorpythonbindings_common_files.cmake
-        Source/Platform/${PAL_PLATFORM_NAME}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
     PLATFORM_INCLUDE_FILES
         Source/Platform/${PAL_PLATFORM_NAME}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
         Source/Platform/Common/${PAL_TRAIT_COMPILER_ID}/editorpythonbindings_static_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake
-    COMPILE_DEFINITIONS
-        PRIVATE
-            PY_PACKAGE="${python_package_name}" 
     INCLUDE_DIRECTORIES
         PRIVATE
             .

Gems/EditorPythonBindings/Code/Source/Platform/Linux/PythonSystemComponent_linux.cpp

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzCore/PlatformDef.h>
-#include <AzCore/std/containers/unordered_set.h>
-#include <AzCore/IO/SystemFile.h>
-#include <AzCore/IO/Path/Path.h>
-#include <AzFramework/StringFunc/StringFunc.h>
-
-namespace Platform
-{
-    bool InsertPythonLibraryPath(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot, const char* subPath)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format(subPath, pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        if (AZ::IO::SystemFile::Exists(libPath.c_str()))
-        {
-            paths.insert(libPath.c_str());
-            return true;
-        }
-        
-        AZ_Warning("python", false, "Python library path should exist! path:%s", libPath.c_str());
-        return false;
-    }
-
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot)
-    {
-        // PY_VERSION_MAJOR_MINOR must be defined through the build scripts based on the current python package (see cmake/LYPython.cmake)
-        #if !defined(PY_VERSION_MAJOR_MINOR)
-        #error "PY_VERSION_MAJOR_MINOR is not defined"
-        #endif
-
-        bool succeeded = true;
-
-        // append lib path to Python paths
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib");
-
-        // append lib-dynload path
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR "/lib-dynload");
-
-        // append base path to dynamic link libraries
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR);
-
-        // append path to site-packages
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/python" PY_VERSION_MAJOR_MINOR "/site-packages");
-        return succeeded;
-    }
-
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format("python/runtime/%s/python", pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-}

Gems/EditorPythonBindings/Code/Source/Platform/Linux/platform_linux_files.cmake

@@ -1,11 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-
-set(FILES
-    ../Linux/PythonSystemComponent_linux.cpp
-)

Gems/EditorPythonBindings/Code/Source/Platform/Mac/PythonSystemComponent_mac.cpp

@@ -1,66 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzCore/PlatformDef.h>
-#include <AzCore/std/containers/unordered_set.h>
-#include <AzCore/IO/SystemFile.h>
-#include <AzCore/IO/Path/Path.h>
-#include <AzFramework/StringFunc/StringFunc.h>
-
-namespace Platform
-{
-
-
-    bool InsertPythonLibraryPath(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot, const char* subPath)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format(subPath, pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        if (AZ::IO::SystemFile::Exists(libPath.c_str()))
-        {
-            paths.insert(libPath.c_str());
-            return true;
-        }
-        
-        AZ_Warning("python", false, "Python library path should exist! path:%s", libPath.c_str());
-        return false;
-    }
-
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot)
-    {
-        // PY_VERSION_MAJOR_MINOR must be defined through the build scripts based on the current python package (see cmake/LYPython.cmake)
-        #if !defined(PY_VERSION_MAJOR_MINOR)
-        #error "PY_VERSION_MAJOR_MINOR is not defined"
-        #endif
-
-        // append lib path to Python paths
-        bool succeeded = true;
-        
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib");
-
-        // append lib-dynload path
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR "/lib-dynload");
-
-        // append base path to dynamic link libraries
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR);
-
-        // append path to site-packages
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR "/lib/python" PY_VERSION_MAJOR_MINOR "/site-packages");
-        return succeeded;
-    }
-
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format("python/runtime/%s/Python.framework/Versions/" PY_VERSION_MAJOR_MINOR, pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-}

Gems/EditorPythonBindings/Code/Source/Platform/Mac/platform_mac_files.cmake

@@ -1,12 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-
-set(FILES
-    ../Mac/PythonSystemComponent_mac.cpp
-)
-

Gems/EditorPythonBindings/Code/Source/Platform/Windows/PythonSystemComponent_windows.cpp

@@ -1,53 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <AzCore/PlatformDef.h>
-#include <AzCore/std/containers/unordered_set.h>
-#include <AzCore/IO/SystemFile.h>
-#include <AzCore/IO/Path/Path.h>
-#include <AzFramework/StringFunc/StringFunc.h>
-
-namespace Platform
-{
-    bool InsertPythonLibraryPath(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot, const char* subPath)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format(subPath, pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        if (AZ::IO::SystemFile::Exists(libPath.c_str()))
-        {
-            paths.insert(libPath.c_str());
-            return true;
-        }
-        
-        AZ_Warning("python", false, "Python library path should exist! path:%s", libPath.c_str());
-        return false;
-    }
-
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        bool succeeded = true;
-        
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/lib/site-packages");
-        succeeded = succeeded && InsertPythonLibraryPath(paths, pythonPackage, engineRoot, "python/runtime/%s/python/DLLs");
-        return succeeded;
-    }
-
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot)
-    {
-        // append lib path to Python paths
-        AZ::IO::FixedMaxPath libPath = engineRoot;
-        libPath /= AZ::IO::FixedMaxPathString::format("python/runtime/%s/python", pythonPackage);
-        libPath = libPath.LexicallyNormal();
-        return libPath.String();
-    }
-}

Gems/EditorPythonBindings/Code/Source/Platform/Windows/platform_windows_files.cmake

@@ -1,11 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-
-set(FILES
-    ../Windows/PythonSystemComponent_windows.cpp
-)

Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp

@@ -16,6 +16,7 @@
 #include <pybind11/eval.h>
 #include <osdefs.h> // for DELIM
 
+
 #include <AzCore/Component/EntityId.h>
 #include <AzCore/IO/SystemFile.h>
 #include <AzCore/Module/DynamicModuleHandle.h>
@@ -38,13 +39,7 @@
 
 #include <AzToolsFramework/API/EditorPythonConsoleBus.h>
 #include <AzToolsFramework/API/EditorPythonScriptNotificationsBus.h>
-
-namespace Platform
-{
-    // Implemented in each different platform's implementation files, as it differs per platform.
-    bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot);
-    AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot);
-}
+#include <AzToolsFramework/API/PythonLoader.h>
 
 // this is called the first time a Python script contains "import azlmbr"
 PYBIND11_EMBEDDED_MODULE(azlmbr, m)
@@ -494,19 +489,31 @@ namespace EditorPythonBindings
         };
 
         // The discovery order will be:
-        //   1 - engine-root/EngineAsets
-        //   2 - gems
-        //   3 - project
-        //   4 - user(dev)
+        //   1 - The python venv site-packages
+        //   2 - engine-root/EngineAsets
+        //   3 - gems
+        //   4 - project
+        //   5 - user(dev)
+
+        // 1 - The python venv site-packages
+        AZ::IO::FixedMaxPath thirdPartyFolder = AzToolsFramework::EmbeddedPython::PythonLoader::GetDefault3rdPartyPath(false);
+
+        AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
+            thirdPartyFolder,
+            AZ::Utils::GetEnginePath().c_str(),
+            [&pythonPathStack](AZ::IO::PathView path)
+            {
+                pythonPathStack.emplace_back(path.Native());
+            });
 
-        // 1 - engine
+        // 2 - engine
         AZ::IO::FixedMaxPath engineRoot;
         if (settingsRegistry->Get(engineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder); !engineRoot.empty())
         {
             resolveScriptPath((engineRoot / "Assets").Native());
         }
 
-        // 2 - gems
+        // 3 - gems
         AZStd::vector<AZ::IO::Path> gemSourcePaths;
         auto AppendGemPaths = [&gemSourcePaths](AZStd::string_view, AZStd::string_view gemPath)
         {
@@ -519,10 +526,10 @@ namespace EditorPythonBindings
             resolveScriptPath(gemSourcePath.Native());
         }
 
-        // 3 - project
+        // 4 - project
         resolveScriptPath(AZStd::string_view{ projectPath });
 
-        // 4 - user
+        // 5 - user
         AZStd::string assetsType;
         AZ::SettingsRegistryMergeUtils::PlatformGet(*settingsRegistry, assetsType,
             AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey, AzFramework::AssetSystem::Assets);
@@ -559,7 +566,7 @@ namespace EditorPythonBindings
         AZ::IO::FixedMaxPath engineRoot = AZ::Utils::GetEnginePath();
 
         // set PYTHON_HOME
-        AZStd::string pyBasePath = Platform::GetPythonHomePath(PY_PACKAGE, engineRoot.c_str());
+        AZStd::string pyBasePath = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonHomePath(engineRoot.c_str()).StringAsPosix();
         if (!AZ::IO::SystemFile::Exists(pyBasePath.c_str()))
         {
             AZ_Warning("python", false, "Python home path must exist! path:%s", pyBasePath.c_str());

Gems/QtForPython/Code/Tests/pyside_auto_menubar_test_case.py

@@ -35,7 +35,7 @@ allWindows = QtGui.QGuiApplication.allWindows()
 printOrExcept(len(allWindows) > 0, 'Value allWindows greater than zero')
 
 azMainWidgetId = azlmbr.qt.QtForPythonRequestBus(azlmbr.bus.Broadcast, 'GetMainWindowId')
-printOrExcept(azMainWidgetId is not 0, 'GetMainWindowId')
+printOrExcept(azMainWidgetId != 0, 'GetMainWindowId')
 
 mainWidgetWindow = QtWidgets.QWidget.find(azMainWidgetId)
 mainWindow = wrapInstance(int(getCppPointer(mainWidgetWindow)[0]), QtWidgets.QMainWindow)

Gems/QtForPython/Editor/Scripts/tests/log_main_window.py

@@ -12,7 +12,7 @@ from PySide2 import QtGui
 from shiboken2 import wrapInstance, getCppPointer
 
 params = azlmbr.qt.QtForPythonRequestBus(azlmbr.bus.Broadcast, 'GetQtBootstrapParameters')
-if(params is not None and params.mainWindowId is not 0):
+if(params is not None and params.mainWindowId != 0):
     mainWidgetWindow = QtWidgets.QWidget.find(params.mainWindowId)
     mainWindow = wrapInstance(int(getCppPointer(mainWidgetWindow)[0]), QtWidgets.QMainWindow)
     mainWindow.menuBar().addMenu("&Hello")

Gems/ScriptCanvas/Assets/TranslationAssets/Classes/SoundAssetRef.names

@@ -24,7 +24,7 @@ the following tools:
 REQUIREMENTS
 ------------
 
- * Python 3.10.5 (64-bit)
+ * Python 3.10.13 (64-bit)
 
 It is recommended that you completely remove any other versions of Python
 installed on your system.

Tools/RemoteConsole/ly_remote_console/README.txt

@@ -14,7 +14,7 @@ read console commands.
 REQUIREMENTS
 ------------
 
- * Python 3.10.5 (64-bit)
+ * Python 3.10.13 (64-bit)
 
 It is recommended that you completely remove any other versions of Python
 installed on your system.

Tools/SerializeContextAnalyzer/SerializeContextAnalyzer.py

@@ -21,7 +21,7 @@ def CreateOutputFolder(path):
         try:
             os.mkdir(folderPath)
         except OSError:
-            print "Failed to create path '{0}'".format(folderPath)
+            print("Failed to create path '{0}'".format(folderPath))
             return False
     return True
 
@@ -81,7 +81,7 @@ def Convert(source, output, template, split, additionalArgs):
     data['Config'] = {}
     data['Config']['Document'] = file
     ParseAdditionalArguments(data['Config'], additionalArgs)
-    print "Template arguments: {}".format(data['Config'])
+    print("Template arguments: {}".format(data['Config']))
 
     if split:
         for c in string.ascii_lowercase:
@@ -117,5 +117,5 @@ if __name__ == "__main__":
         templatePath = os.path.abspath(templatePath)
 
     if CreateOutputFolder(outputPath):
-        print 'Converting from "{0}" to "{1}" using template "{2}".'.format(sourcePath, outputPath, templatePath)
+        print('Converting from "{0}" to "{1}" using template "{2}".'.format(sourcePath, outputPath, templatePath))
         Convert(sourcePath, outputPath, templatePath, args.split, remainingArgs)

Tools/styleui/styleui.py

@@ -126,30 +126,28 @@ def make_copy(args):
     ui_content = read_ui_file(args.ui_file_path)
     (inserted_variables, inserted_qss) = insert_styles_into_ui(qss_content, variables_content, ui_content)
     ui_copy_file_path = write_ui_copy(ui_content)
-    print 'copied', args.ui_file_path, 'to', ui_copy_file_path, 'with styles from', args.qss_file_path
+    print(f"copied {args.ui_file_path} to {ui_copy_file_path} with styles from {args.qss_file_path}")
     return (ui_copy_file_path, inserted_variables, inserted_qss)
 
 def read_qss_file(qss_file_path):
     with open(qss_file_path, 'r') as qss_file:
         qss_content = qss_file.read()
-        #print 'qss_content', qss_content
         return qss_content
 
 def read_variables_file(variables_file_path):
     with open(variables_file_path, 'r') as variables_file:
         variables_content = json.load(variables_file)
-        #print 'variables_content', variables_content
         return variables_content
         
 def replace_variables(qss_content, variables_content):
     for name, value in variables_content.get('StylesheetVariables', {}).iteritems():
         qss_content = qss_content.replace('@' + name, '/*@' + name + '*/' + value + '/*@*/')
-    #print 'replace_variables', qss_content
+
     return qss_content
     
 def read_ui_file(ui_file_path):
     ui_content = ET.parse(ui_file_path)
-    #print 'ui_content', ui_content
+
     return ui_content
     
 def insert_styles_into_ui(qss_content, variables_content, ui_content):
@@ -179,13 +177,11 @@ def remove_variables_from_property_value(property_value):
         if end_index != -1:
             removed_variables = property_value[start_index + len(START_VARIABLES_MARKER):end_index]
             property_value = property_value[:start_index] + property_value[end_index + len(END_VARIABLES_MARKER):]
-            #print 'removed', property_value
     return (property_value, removed_variables)
 
 def insert_variables_into_property_value(property_value, variables_content):    
     inserted_variables = json.dumps(variables_content, sort_keys=True, indent=4)
     property_value = property_value + START_VARIABLES_MARKER + inserted_variables + END_VARIABLES_MARKER
-    #print 'inserted', property_value
     return (property_value, inserted_variables)
     
 START_QSS_MARKER = '\n/*** START INSERTED STYLES ***/\n'
@@ -199,23 +195,20 @@ def remove_qss_from_property_value(property_value):
         if end_index != -1:
             removed_qss = property_value[start_index + len(START_QSS_MARKER):end_index]
             property_value = property_value[:start_index] + property_value[end_index + len(END_QSS_MARKER):]
-            #print 'removed', property_value
     return (property_value, removed_qss)
 
 def insert_qss_into_property_value(property_value, qss_content):    
     property_value = property_value + START_QSS_MARKER + qss_content + END_QSS_MARKER
-    #print 'inserted', property_value
     return (property_value, qss_content)
     
 def write_ui_copy(ui_content):
     (ui_copy_file, ui_copy_file_path) = tempfile.mkstemp(suffix='.ui', text=True)
-    #print 'ui_copy_file_path', ui_copy_file_path
+
     ui_content.write(os.fdopen(ui_copy_file, 'w'))
-    #os.close(ui_copy_file) ElementTree.write must be closing... fails if called
     return ui_copy_file_path
 
 def start_monitoring_for_changes(ui_copy_file_path, args, inserted_variables, inserted_qss):
-    print 'monitoring', ui_copy_file_path, 'for changes'
+    print(f'monitoring {ui_copy_file_path} for changes')
     monitor_thread = Thread(target = monitor_for_changes, args=(ui_copy_file_path, args, inserted_variables, inserted_qss))
     monitor_thread.start()
     return monitor_thread
@@ -254,57 +247,56 @@ def remove_styles_from_ui(ui_content):
     return (removed_variables, removed_qss)
     
 def update_ui(ui_content, ui_file_path):
-    print 'updating', ui_file_path, 'with changes'
+    print(f'updating {ui_file_path} with changes')
     try:
         ui_content.write(ui_file_path)
     except Exception as e:
-        print '\n*** WRITE FAILED', e
+        print(f'\n*** WRITE FAILED {e}')
         parts = os.path.splitext(ui_file_path)
         temp_path = parts[0] + '_BACKUP' + parts[1]
-        print '*** saving to', temp_path, 'instead\n'
+        print(f'*** saving to {temp_path} instead\n')
         ui_content.write(temp_path)
     
 def update_variables(removed_variables, variables_file_path):
-    print 'updating', variables_file_path, 'with changes'
+    print(f'updating {variables_file_path} with changes')
     try:
         with open(variables_file_path, "w") as variables_file:
             variables_file.write(removed_variables)
     except Exception as e:
-        print '\n*** WRITE FAILED', e
+        print(f'\n*** WRITE FAILED {e}')
         parts = os.path.splitext(variables_file_path)
         temp_path = parts[0] + '_BACKUP' + parts[1]
-        print '*** saving to', temp_path, 'instead\n'
+        print(f'*** saving to {temp_path} instead\n')
         with open(temp_path, "w") as variables_file:
             variables_file.write(removed_variables)
     
 def update_qss(removed_qss, qss_file_path):
-    print 'updating', qss_file_path, 'with changes'
+    print(f'updating {qss_file_path} with changes')
     removed_qss = re.sub(r'/\*@(\w+)\*/.*/\*@\*/', '@\g<1>', removed_qss)
     try:
         with open(qss_file_path, "w") as qss_file:
             qss_file.write(removed_qss)
     except Exception as e:
-        print '\n*** WRITE FAILED', e
+        print(f'\n*** WRITE FAILED {e}')
         parts = os.path.splitext(qss_file_path)
         temp_path = parts[0] + '_BACKUP' + parts[1]
-        print '*** saving to', temp_path, 'instead\n'
+        print(f'*** saving to {temp_path} instead\n')
         with open(temp_path, "w") as qss_file:
             qss_file.write(removed_qss)
     
 def stop_monitor_for_changes(monitor_thread):
-    print 'stopping change monitor'
+    print(f'stopping change monitor')
     global continue_monitoring
     continue_monitoring = False
     monitor_thread.join()
     
 def run_designer(designer_file_path, ui_copy_file_path):
-    print 'starting designer with', ui_copy_file_path
+    print(f'starting designer with {ui_copy_file_path}')
     subprocess.call([designer_file_path, ui_copy_file_path])
-    print 'designer exited'
+    print(f'designer exited')
     
 def delete_copy(ui_copy_file_path):
-    print 'deleting', ui_copy_file_path
+    print(f'deleting {ui_copy_file_path}')
     os.remove(ui_copy_file_path)
     
 main()
-

cmake/3rdParty/Platform/Linux/Python_linux_aarch64.cmake

@@ -0,0 +1,28 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Setup the Python global variables and download and associate python to the correct package
+
+# Python package info
+ly_set(LY_PYTHON_VERSION 3.10.13)
+ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
+ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.13-rev2-linux-aarch64)
+ly_set(LY_PYTHON_PACKAGE_HASH 30bc2731e2ac54d8e22d36ab15e30b77aefe2dce146ef92d6f20adc0a9c5b14e)
+
+# Python package relative paths
+ly_set(LY_PYTHON_BIN_PATH "python/bin")
+ly_set(LY_PYTHON_LIB_PATH "python/lib")
+ly_set(LY_PYTHON_EXECUTABLE "python")
+
+# Python venv relative paths
+ly_set(LY_PYTHON_VENV_BIN_PATH "bin")
+ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
+ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
+ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
+
+ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})

cmake/3rdParty/Platform/Linux/Python_linux_x86_64.cmake

@@ -0,0 +1,28 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Setup the Python global variables and download and associate python to the correct package
+
+# Python package info
+ly_set(LY_PYTHON_VERSION 3.10.13)
+ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
+ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.13-rev2-linux)
+ly_set(LY_PYTHON_PACKAGE_HASH a7832f9170a3ac93fbe678e9b3d99a977daa03bb667d25885967e8b4977b86f8)
+
+# Python package relative paths
+ly_set(LY_PYTHON_BIN_PATH "python/bin")
+ly_set(LY_PYTHON_LIB_PATH "python/lib")
+ly_set(LY_PYTHON_EXECUTABLE "python")
+
+# Python venv relative paths
+ly_set(LY_PYTHON_VENV_BIN_PATH "bin")
+ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
+ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
+ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
+
+ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})

cmake/3rdParty/Platform/Linux/cmake_linux_files.cmake

@@ -10,4 +10,6 @@ set(FILES
     BuiltInPackages_linux_aarch64.cmake
     BuiltInPackages_linux_x86_64.cmake
     Wwise_linux.cmake
+    Python_linux_aarch64.cmake
+    Python_linux_x86_64.cmake
 )

cmake/3rdParty/Platform/Mac/Python_mac.cmake

@@ -0,0 +1,28 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Setup the Python global variables and download and associate python to the correct package
+
+# Python package info
+ly_set(LY_PYTHON_VERSION 3.10.13)
+ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
+ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.13-rev1-darwin)
+ly_set(LY_PYTHON_PACKAGE_HASH 14a88370fa8673344cf51354ce1a1021a0a1fb1742d774b5a0033a94e3384ec1)
+
+# Python package relative paths
+ly_set(LY_PYTHON_BIN_PATH "Python.framework/Versions/Current/bin/")
+ly_set(LY_PYTHON_LIB_PATH "Python.framework/Versions/Current/lib")
+ly_set(LY_PYTHON_EXECUTABLE "python3")
+
+# Python venv relative paths
+ly_set(LY_PYTHON_VENV_BIN_PATH "bin")
+ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
+ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
+ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
+
+ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})

cmake/3rdParty/Platform/Mac/cmake_mac_files.cmake

@@ -10,4 +10,5 @@ set(FILES
     BuiltInPackages_mac.cmake
     OpenGLInterface_mac.cmake
     Wwise_mac.cmake
+    Python_mac.cmake
 )

cmake/3rdParty/Platform/Windows/Python_windows.cmake

@@ -0,0 +1,28 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Setup the Python global variables and download and associate python to the correct package
+
+# Python package info
+ly_set(LY_PYTHON_VERSION 3.10.13)
+ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
+ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.13-rev1-windows)
+ly_set(LY_PYTHON_PACKAGE_HASH a6f1fd50552c8780852b75186a97469f77d55d06a01de73e5ca601e4a12be232)
+
+# Python package relative paths
+ly_set(LY_PYTHON_BIN_PATH "python")
+ly_set(LY_PYTHON_LIB_PATH "Lib")
+ly_set(LY_PYTHON_EXECUTABLE "python.exe")
+
+# Python venv relative paths
+ly_set(LY_PYTHON_VENV_BIN_PATH "Scripts")
+ly_set(LY_PYTHON_VENV_LIB_PATH "Lib")
+ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/site-packages")
+ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python.exe")
+
+ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})

cmake/3rdParty/Platform/Windows/cmake_windows_files.cmake

@@ -9,4 +9,5 @@
 set(FILES
     BuiltInPackages_windows.cmake
     Wwise_windows.cmake
+    Python_windows.cmake
 )

cmake/CalculateEnginePathId.cmake

@@ -0,0 +1,35 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# This script will generate a unique ID of the current engine path by using the first 9 characters of the SHA1 hash of the absolute engine path
+# and write the results to STDOUT
+
+
+if(${CMAKE_ARGC} LESS 3)
+    message(FATAL_ERROR "Missing required engine path argument to calculate id from.")
+endif()
+
+# Get and normalize the path passed into this script as the main argument
+set(PATH_TO_HASH ${CMAKE_ARGV3})
+cmake_path(NORMAL_PATH PATH_TO_HASH)
+
+# Sanity check to make sure this is the path to the engine
+set(ENGINE_SANITY_CHECK_FILE "${PATH_TO_HASH}/engine.json")
+if (NOT EXISTS "${ENGINE_SANITY_CHECK_FILE}")
+    message(FATAL_ERROR "Path ${PATH_TO_HASH} is not a valid engine path.")
+endif()
+
+string(TOLOWER ${PATH_TO_HASH} PATH_TO_HASH)
+
+# Calculate the path id based on the first 8 characters of the SHA1 hash of the normalized path
+string(SHA1 ENGINE_SOURCE_PATH_HASH "${PATH_TO_HASH}")
+string(SUBSTRING ${ENGINE_SOURCE_PATH_HASH} 0 8 ENGINE_SOURCE_PATH_ID)
+
+# Note: using 'message(STATUS ..' will print to STDOUT, but will always include a double hyphen '--'. Instead we will 
+# use the cmake echo command directly to do this
+execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${ENGINE_SOURCE_PATH_ID})

cmake/LYPython.cmake

@@ -9,57 +9,120 @@
 include_guard()
 
 include(cmake/LySet.cmake)
+include(cmake/3rdPartyPackages.cmake)
 
 # this script exists to make sure a python interpreter is immediately available
 # it will both locate and run pip on python for our requirements.txt
 # but you can also call update_pip_requirements(filename) at any time after.
 
-# this is different from the usual package usage, because even if we are targetting
+# this is different from the usual package usage, because even if we are targeting
 # android, for example, we may still be doing so on a windows HOST pc, and the
 # python interpreter we want to use is for the windows HOST pc, not the PAL platform:
 
-# CMAKE_HOST_SYSTEM_NAME is  "Windows", "Darwin", or "Linux" in our cases..
-if (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Linux" )
 
-    # Note: CMAKE_HOST_SYSTEM_PROCESSOR may not available in this script if it is not
-    #       invoked from the base CMakeList.txt since project needs to be declared.
-    #       We will extract the host architecture manually if this script is called externally
-    if (${CMAKE_SYSTEM_ARCHITECTURE})
-        set(LINUX_HOST_ARCHITECTURE ${CMAKE_SYSTEM_ARCHITECTURE})
-    else()
-        execute_process(COMMAND uname -m OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LINUX_HOST_ARCHITECTURE)
-    endif()
+set(LY_PAL_PYTHON_PACKAGE_FILE_NAME ${LY_ROOT_FOLDER}/cmake/3rdParty/Platform/${PAL_HOST_PLATFORM_NAME}/Python_${PAL_HOST_PLATFORM_NAME_LOWERCASE}${LY_HOST_ARCHITECTURE_NAME_EXTENSION}.cmake)
+cmake_path(NORMAL_PATH LY_PAL_PYTHON_PACKAGE_FILE_NAME)
+include(${LY_PAL_PYTHON_PACKAGE_FILE_NAME})
+
+# settings and globals
+ly_set(LY_PYTHON_DEFAULT_REQUIREMENTS_TXT "${LY_ROOT_FOLDER}/python/requirements.txt")
+
+# The expected venv for python is based on an ID generated based on the path of the current
+# engine using the logic in cmake/CalculateEnginePathId.cmake. 
+execute_process(COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CalculateEnginePathId.cmake "${CMAKE_CURRENT_SOURCE_DIR}/"
+                OUTPUT_VARIABLE ENGINE_SOURCE_PATH_ID
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# Normalize the expected location of the Python venv and set its path globally
+set(PYTHON_VENV_PATH "${LY_3RDPARTY_PATH}/venv/${ENGINE_SOURCE_PATH_ID}")
+cmake_path(NORMAL_PATH PYTHON_VENV_PATH )
+ly_set(LY_PYTHON_VENV_PATH ${PYTHON_VENV_PATH})
+
+# On Linux systems, we need to create a symlink to the shared library as well
+# due to the fact that a symlink is created for the python executable inside 
+# the virtual environment, but the python executable is custom built to use
+# an RPATH set to $ORIGIN/../lib to match the install structure in the package.
+if ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux")
+    set(LY_LINK_TO_SHARED_LIBRARY TRUE)
+else()
+    set(LY_LINK_TO_SHARED_LIBRARY FALSE)
+endif()
 
-    if (${LINUX_HOST_ARCHITECTURE} STREQUAL "x86_64")
-        ly_set(LY_PYTHON_VERSION 3.10.5)
-        ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
-        ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.5-rev4-linux)
-        ly_set(LY_PYTHON_PACKAGE_HASH 0144a4a5c9d39a834a319c98ce021741dac579bc39f60a08b6784b71d0983462)
-    elseif(${LINUX_HOST_ARCHITECTURE} STREQUAL "aarch64")
-        ly_set(LY_PYTHON_VERSION 3.10.5)
-        ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
-        ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.5-rev4-linux-aarch64)
-        ly_set(LY_PYTHON_PACKAGE_HASH c6e5e9cecad36ee1baa3bcbec629fb1cdeb6854f73132e0ca2d97f5b3b8b3a0e)
+
+function(ly_setup_python_venv)
+
+    # Check if we need to setup a new venv.
+    set(CREATE_NEW_VENV FALSE)
+    # We track if an existing venv matches the current version of the Python 3rd Party Package
+    # by tracking the package hash within the venv folder. If there it is missing or does not
+    # match the current package hash ${LY_PYTHON_PACKAGE_HASH} then reset the venv and request
+    # that a new venv is created with the current Python package at :
+    # 
+    if (EXISTS "${PYTHON_VENV_PATH}/.hash")
+        file(READ "${PYTHON_VENV_PATH}/.hash" LY_CURRENT_VENV_PACKAGE_HASH
+             LIMIT 80)
+        if (NOT ${LY_CURRENT_VENV_PACKAGE_HASH} STREQUAL ${LY_PYTHON_PACKAGE_HASH})
+            # The package hash changed, re-install the venv
+            set(CREATE_NEW_VENV TRUE)
+            file(REMOVE_RECURSE "${PYTHON_VENV_PATH}")
+        else()
+            # Sanity check to make sure the python launcher in the venv works
+            execute_process(COMMAND ${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_PYTHON} --version
+                            RESULT_VARIABLE command_result)
+            if (NOT ${command_result} EQUAL 0)
+                message(STATUS "Error validating python inside the venv. Reinstalling")
+                set(CREATE_NEW_VENV TRUE)
+                file(REMOVE_RECURSE "${PYTHON_VENV_PATH}")
+            else()
+                message(STATUS "Using Python venv at ${PYTHON_VENV_PATH}")
+            endif()
+        endif()
     else()
-        message(FATAL_ERROR "Linux host architecture ${LINUX_HOST_ARCHITECTURE} not supported.")
+        set(CREATE_NEW_VENV TRUE)
     endif()
 
-elseif  (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Darwin" )
-    ly_set(LY_PYTHON_VERSION 3.10.5)
-    ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
-    ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.5-rev2-darwin)
-    ly_set(LY_PYTHON_PACKAGE_HASH 46d7c74c64bf639279c53a68ff958d9955e01e08d293524958eb7ea7cac9c4c5)
-elseif  (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Windows" )
-    ly_set(LY_PYTHON_VERSION 3.10.5)
-    ly_set(LY_PYTHON_VERSION_MAJOR_MINOR 3.10)
-    ly_set(LY_PYTHON_PACKAGE_NAME python-3.10.5-rev1-windows)
-    ly_set(LY_PYTHON_PACKAGE_HASH c012e7c8fd20e632446d2cd689a9472e4e4495da7534d484d0f1c63840222cbb)
-endif()
+    if (CREATE_NEW_VENV)
+        # Run the install venv command, but skip the pip setup because we may need to manually create 
+        # a link to the shared library within the created virtual environment before proceeding with
+        # the pip install command.
+        message(STATUS "Creating Python venv at ${PYTHON_VENV_PATH}")
+        execute_process(COMMAND "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}/${LY_PYTHON_EXECUTABLE}" -m venv "${PYTHON_VENV_PATH}" --without-pip --clear
+                        WORKING_DIRECTORY "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}"
+                        COMMAND_ECHO STDOUT
+                        RESULT_VARIABLE command_result)
+
+        if (NOT ${command_result} EQUAL 0)
+            message(FATAL_ERROR "Error creating a venv")
+        endif()
 
-# settings and globals
-ly_set(LY_PYTHON_DEFAULT_REQUIREMENTS_TXT "${LY_ROOT_FOLDER}/python/requirements.txt")
+        if (LY_LINK_TO_SHARED_LIBRARY)
+            # Create a symlink to the package's python shared library inside the virtual environments
+            # own lib sub folder to match the original package structure
+            execute_process(COMMAND ln -s -f "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_LIB_PATH}/libpython3.10.so.1.0" libpython3.10.so.1.0
+                            WORKING_DIRECTORY "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_LIB_PATH}"
+                            COMMAND_ECHO STDOUT
+                            RESULT_VARIABLE command_result)
+            if (NOT ${command_result} EQUAL 0)
+                message(WARNING "Unable to createa venv shared library link.")
+            endif()
+        endif()
 
-include(cmake/3rdPartyPackages.cmake)
+        # Manually install pip into the virtual environment
+        message(STATUS "Installing pip into venv at ${PYTHON_VENV_PATH} (${PYTHON_VENV_PATH}/${LY_PYTHON_BIN_PATH})")
+
+        execute_process(COMMAND "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_PYTHON}" -m ensurepip --upgrade --default-pip -v
+                        WORKING_DIRECTORY "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_BIN_PATH}"
+                        COMMAND_ECHO STDOUT
+                        RESULT_VARIABLE command_result)
+
+        if (NOT ${command_result} EQUAL 0)
+            message(FATAL_ERROR "Error installing pip into venv: ${LY_PIP_ERROR}")
+        endif()
+
+        file(WRITE "${PYTHON_VENV_PATH}/.hash" ${LY_PYTHON_PACKAGE_HASH})
+    endif()
+    
+endfunction()
 
 # update_pip_requirements
 #    param: requirements_file_path = path to a requirements.txt file.
@@ -69,6 +132,7 @@ include(cmake/3rdPartyPackages.cmake)
 # file, and should be unique to your particular gem/package/3rdParty.  It will be
 # used as a file name, so avoid special characters that would fail as a file name.
 function(update_pip_requirements requirements_file_path unique_name)
+
     # we run with --no-deps to prevent it from cascading to child dependencies 
     # and getting more than we expect.
     # to produce a new requirements.txt use pip-compile from pip-tools alongside pip freeze
@@ -76,7 +140,7 @@ function(update_pip_requirements requirements_file_path unique_name)
 
     # We skip running requirements.txt if we can (we use a stamp file to keep track)
     # the stamp file is kept in the binary folder with a similar file path to the source file.
-    set(stamp_file ${CMAKE_BINARY_DIR}/packages/requirements_files/${unique_name}.stamp)
+    set(stamp_file ${LY_PYTHON_VENV_PATH}/requirements_files/${unique_name}.stamp)
     get_filename_component(stamp_file_directory ${stamp_file} DIRECTORY)
     file(MAKE_DIRECTORY ${stamp_file_directory})
 
@@ -100,9 +164,10 @@ function(update_pip_requirements requirements_file_path unique_name)
     endif()
 
     set(ENV{PYTHONNOUSERSITE} 1)
+
     execute_process(COMMAND 
         ${LY_PYTHON_CMD} -m pip install -r "${requirements_file_path}" --disable-pip-version-check --no-warn-script-location
-        WORKING_DIRECTORY ${Python_BINFOLDER}
+        WORKING_DIRECTORY ${LY_ROOT_FOLDER}/python
         RESULT_VARIABLE PIP_RESULT
         OUTPUT_VARIABLE PIP_OUT 
         ERROR_VARIABLE PIP_OUT
@@ -136,7 +201,7 @@ function(update_pip_requirements requirements_file_path unique_name)
 
     execute_process(COMMAND 
         ${LY_PYTHON_CMD} -m pip check
-        WORKING_DIRECTORY ${Python_BINFOLDER} 
+        WORKING_DIRECTORY ${LY_ROOT_FOLDER}/python
         RESULT_VARIABLE PIP_RESULT
         OUTPUT_VARIABLE PIP_OUT 
         ERROR_VARIABLE PIP_OUT
@@ -160,7 +225,7 @@ endfunction()
 # the pip_package_name should be the name given to the package in setup.py so that
 # any old versions may be uninstalled using setuptools before we install the new one.
 function(ly_pip_install_local_package_editable package_folder_path pip_package_name)
-    set(stamp_file ${CMAKE_BINARY_DIR}/packages/pip_installs/${pip_package_name}.stamp)
+    set(stamp_file ${LY_PYTHON_VENV_PATH}/packages/pip_installs/${pip_package_name}.stamp)
     get_filename_component(stamp_file_directory ${stamp_file} DIRECTORY)
     file(MAKE_DIRECTORY ${stamp_file_directory})
     
@@ -196,19 +261,21 @@ function(ly_pip_install_local_package_editable package_folder_path pip_package_n
     # uninstall any old versions of this package first:
     execute_process(COMMAND 
         ${LY_PYTHON_CMD} -m pip uninstall ${pip_package_name} -y  --disable-pip-version-check 
-        WORKING_DIRECTORY ${Python_BINFOLDER}
+        WORKING_DIRECTORY ${LY_ROOT_FOLDER}/python
         RESULT_VARIABLE PIP_RESULT
         OUTPUT_VARIABLE PIP_OUT 
         ERROR_VARIABLE PIP_OUT
         )
+
     # we discard the error output of above, since it might not be installed, which is ok
     message(VERBOSE "pip uninstall result: ${PIP_RESULT}")
     message(VERBOSE "pip uninstall output: ${PIP_OUT}")
 
     # now install the new one:
+
     execute_process(COMMAND 
         ${LY_PYTHON_CMD} -m pip install -e ${package_folder_path} --no-deps --disable-pip-version-check  --no-warn-script-location 
-        WORKING_DIRECTORY ${Python_BINFOLDER}
+        WORKING_DIRECTORY ${LY_ROOT_FOLDER}/python
         RESULT_VARIABLE PIP_RESULT
         OUTPUT_VARIABLE PIP_OUT 
         ERROR_VARIABLE PIP_OUT
@@ -237,49 +304,28 @@ endfunction()
 # But we don't want to full verify using hashes after we successfully get it the
 # first time.
 
-set(temp_LY_PACKAGE_VALIDATE_PACKAGE ${LY_PACKAGE_VALIDATE_PACKAGE})
-if (EXISTS  ${LY_ROOT_FOLDER}/python/runtime/${LY_PYTHON_PACKAGE_NAME})
-    # we will not validate the hash of every file, just that it is present
-    # this is not just an optimization, see comment above.
-    set(LY_PACKAGE_VALIDATE_PACKAGE FALSE)
-endif()
 
+# We need to download the associated Python package early and install the venv 
 ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
-ly_set_package_download_location(${LY_PYTHON_PACKAGE_NAME} ${LY_ROOT_FOLDER}/python/runtime)
 ly_download_associated_package(Python)
-
-ly_set(LY_PACKAGE_VALIDATE_CONTENTS ${temp_LY_PACKAGE_VALIDATE_CONTENTS})
-ly_set(LY_PACKAGE_VALIDATE_PACKAGE ${temp_LY_PACKAGE_VALIDATE_PACKAGE})
+ly_setup_python_venv()
 
 if (NOT CMAKE_SCRIPT_MODE_FILE)
-    # if we're in script mode, we dont want to actually try to find package or anything else
 
-    find_package(Python ${LY_PYTHON_VERSION} REQUIRED)
 
     # note - if you want to use a normal python via FindPython instead of the LY package above,
     # you may have to declare the below variables after find_package, as the project scripts are 
     # looking for the below variables specifically.
+    find_package(Python ${LY_PYTHON_VERSION} REQUIRED)
 
     # verify the required variables are present:
-    if (NOT Python_EXECUTABLE OR NOT Python_HOME OR NOT Python_PATHS)
+    if (NOT EXISTS "${PYTHON_VENV_PATH}/" OR NOT Python_EXECUTABLE OR NOT Python_HOME OR NOT Python_PATHS)
         message(SEND_ERROR "Python installation not valid expected to find all of the following variables set:")
         message(STATUS "    Python_EXECUTABLE:  ${Python_EXECUTABLE}")
         message(STATUS "    Python_HOME:        ${Python_HOME}")
         message(STATUS "    Python_PATHS:       ${Python_PATHS}")
     else()
-        message(STATUS "Using Python ${Python_VERSION} at ${Python_EXECUTABLE}")
-        message(VERBOSE "    Python_EXECUTABLE:  ${Python_EXECUTABLE}")
-        message(VERBOSE "    Python_HOME:        ${Python_HOME}")
-        message(VERBOSE "    Python_PATHS:       ${Python_PATHS}")
-
-        get_filename_component(Python_BINFOLDER ${Python_EXECUTABLE} DIRECTORY)
-
-        # make sure other utils can find python / pip / etc
-        # using find_program:
-        LIST(APPEND CMAKE_PROGRAM_PATH "${Python_BINFOLDER}")
-        
-        # some platforms have this scripts dir, its harmless to add for those that do not.
-        LIST(APPEND CMAKE_PROGRAM_PATH "${Python_BINFOLDER}/Scripts")   
+        LIST(APPEND CMAKE_PROGRAM_PATH "${LY_ROOT_FOLDER}/python")
 
         # those using python should call it via LY_PYTHON_CMD - it adds the extra "-s" param
         # this param causes python to ignore the users profile folder which can have bogus

cmake/Platform/Linux/Packaging/postinst.in

@@ -15,8 +15,8 @@ set -o errexit # exit on the first failure encountered
     ln -s @CPACK_PACKAGING_INSTALL_PREFIX@/bin/Linux/profile/Default/Editor /usr/local/bin/o3de.editor
        
     pushd @CPACK_PACKAGING_INSTALL_PREFIX@
-    python/get_python.sh
     chown -R $SUDO_USER .
+    sudo -u $SUDO_USER python/get_python.sh
     popd
 
 } &> /dev/null # hide output

cmake/Platform/Linux/Packaging/prerm.in

@@ -11,8 +11,12 @@ set -o errexit # exit on the first failure encountered
 
 {
     pushd @CPACK_PACKAGING_INSTALL_PREFIX@
-    # delete python downloads
-    rm -rf python/downloaded_packages python/runtime
+
+    # delete __pycache__
+    rm -rf scripts/o3de/o3de/__pycache__
+    # delete the user folder
+    rm -rf bin/Linux/profile/Default/user
+    # delete any generated egg-link files
     find . -type d -name *.egg-info -prune -exec rm -rf {} \;
     popd
 } &> /dev/null # hide output

cmake/Platform/Linux/Packaging_Snapcraft.cmake

@@ -23,8 +23,20 @@ endif()
 
 execute_process (COMMAND lsb_release -a)
 
+# Download python before packaging. The python runtimes themselves will not be part of the packaging, but
+# the following script will also run through the scripts and create the necessary egg-link folders
+# which will be installed into the package
 execute_process (COMMAND bash ${CPACK_TEMPORARY_DIRECTORY}/O3DE/${CPACK_PACKAGE_VERSION}/python/get_python.sh)
 
+# Since snap is immutable, we will pre-compile all the python scripts that is packaged into python byte-codes
+set(O3DE_PY_SUBDIRS "Code;Gems;scripts;Tools")
+foreach(engine_subfolder ${O3DE_PY_SUBDIRS})
+    file(GLOB_RECURSE pyfiles "${CPACK_TEMPORARY_DIRECTORY}/O3DE/${CPACK_PACKAGE_VERSION}/${engine_subfolder}/*.py")
+    foreach(pyfile ${pyfiles})
+        execute_process (COMMAND bash ${CPACK_TEMPORARY_DIRECTORY}/O3DE/${CPACK_PACKAGE_VERSION}/python/python.sh -m py_compile ${pyfile})
+    endforeach()
+endforeach()
+
 # make sure that all files have the correct permissions
 execute_process (COMMAND chmod -R 755 O3DE
                  WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY}

cmake/cmake_files.cmake

@@ -10,6 +10,7 @@ set(FILES
     3rdParty.cmake
     3rdPartyPackages.cmake
     AzAutoGen.py
+    CalculateEnginePathId.cmake
     CMakeFiles.cmake
     CommandExecution.cmake
     Configurations.cmake