Pārlūkot izejas kodu

Updates to fix crash in O3DE caused by the python relocation (#17623)

1. Relocation of Python Packages and venv to ~/.o3de/Python (Independent of the 3rdParty folder)
2. Symlinking python3.10.so instead of doing a copy
3. Extra PhysX 4/5 Cleanup

Signed-off-by: Steve Pham <[email protected]>
Steve Pham 1 gadu atpakaļ
vecāks
revīzija
b8114db595
36 mainītis faili ar 212 papildinājumiem un 242 dzēšanām
  1. 8 0
      Code/Framework/AzCore/AzCore/Utils/Utils.cpp
  2. 5 0
      Code/Framework/AzCore/AzCore/Utils/Utils.h
  3. 5 5
      Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp
  4. 10 0
      Code/Framework/AzCore/Platform/Windows/AzCore/Utils/Utils_Windows.cpp
  5. 41 64
      Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.cpp
  6. 8 11
      Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.h
  7. 0 11
      Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Linux.h
  8. 0 10
      Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Platform.h
  9. 5 3
      Code/Framework/AzToolsFramework/Platform/Linux/platform_linux.cmake
  10. 0 2
      Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake
  11. 0 11
      Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Mac.h
  12. 0 10
      Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Platform.h
  13. 4 0
      Code/Framework/AzToolsFramework/Platform/Mac/platform_mac.cmake
  14. 0 2
      Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake
  15. 0 10
      Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Platform.h
  16. 0 11
      Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Windows.h
  17. 0 2
      Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake
  18. 18 18
      Code/Framework/AzToolsFramework/Tests/PythonLoaderTests.cpp
  19. 1 3
      Code/Tools/ProjectManager/Source/PythonBindings.cpp
  20. 0 3
      Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp
  21. 0 7
      cmake/3rdParty/Platform/Linux/BuiltInPackages_linux_aarch64.cmake
  22. 0 7
      cmake/3rdParty/Platform/Linux/BuiltInPackages_linux_x86_64.cmake
  23. 12 1
      cmake/3rdParty/Platform/Linux/Python_linux_aarch64.cmake
  24. 12 1
      cmake/3rdParty/Platform/Linux/Python_linux_x86_64.cmake
  25. 0 7
      cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake
  26. 3 1
      cmake/3rdParty/Platform/Mac/Python_mac.cmake
  27. 3 1
      cmake/3rdParty/Platform/Windows/Python_windows.cmake
  28. 29 4
      cmake/3rdPartyPackages.cmake
  29. 20 24
      cmake/LYPython.cmake
  30. 11 0
      cmake/Platform/Common/RuntimeDependencies_common.cmake
  31. 13 1
      cmake/Platform/Linux/runtime_dependencies_linux.cmake.in
  32. 0 2
      python/.gitignore
  33. 0 1
      python/.p4ignore
  34. 1 1
      python/python.cmd
  35. 2 7
      python/python.sh
  36. 1 1
      scripts/build/ci_build.py

+ 8 - 0
Code/Framework/AzCore/AzCore/Utils/Utils.cpp

@@ -428,6 +428,14 @@ namespace AZ::Utils
         return AZ::Success(thirdPartyFolder);
     }
 
+    AZ::IO::FixedMaxPathString GetO3dePythonVenvRoot(AZ::SettingsRegistryInterface* settingsRegistry /*= nullptr*/)
+    {
+        // Locate the manifest directory
+        auto manifestDirectory = AZ::IO::FixedMaxPath(GetO3deManifestDirectory(settingsRegistry)) / "Python" / "venv";
+        return manifestDirectory.Native();
+    }
+
+
     template AZ::Outcome<AZStd::string, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<int8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<uint8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);

+ 5 - 0
Code/Framework/AzCore/AzCore/Utils/Utils.h

@@ -176,6 +176,11 @@ namespace AZ
         //! Returns the outcome of the request, the configured 3rd Party path if successful, the error message if not.
         AZ::Outcome<AZStd::string, AZStd::string> Get3rdPartyDirectory(AZ::SettingsRegistryInterface* settingsRegistry = nullptr);
 
+        //! Retrieves the full directory to the O3DE Python virtual environment (venv) folder. 
+        //! @param settingsRegistry pointer to the SettingsRegistry to use for lookup
+        //! If nullptr, the AZ::Interface instance of the SettingsRegistry is used
+        AZ::IO::FixedMaxPathString GetO3dePythonVenvRoot(AZ::SettingsRegistryInterface* settingsRegistry = nullptr);
+
 
         //! error code value returned when GetEnv fails
         enum class GetEnvErrorCode

+ 5 - 5
Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp

@@ -39,16 +39,16 @@ namespace AZ::Utils
             }
         }
 
-        if (const char* homePath = std::getenv("HOME"); homePath != nullptr)
+        struct passwd* pass = getpwuid(getuid());
+        if (pass)
         {
-            AZ::IO::FixedMaxPath path{homePath};
+            AZ::IO::FixedMaxPath path{pass->pw_dir};
             return path.Native();
         }
 
-        struct passwd* pass = getpwuid(getuid());
-        if (pass)
+        if (const char* homePath = std::getenv("HOME"); homePath != nullptr)
         {
-            AZ::IO::FixedMaxPath path{pass->pw_dir};
+            AZ::IO::FixedMaxPath path{homePath};
             return path.Native();
         }
 

+ 10 - 0
Code/Framework/AzCore/Platform/Windows/AzCore/Utils/Utils_Windows.cpp

@@ -7,10 +7,12 @@
  */
 
 #include <AzCore/Utils/Utils.h>
+#include <AzCore/IO/Path/Path.h>
 #include <AzCore/PlatformIncl.h>
 #include <AzCore/std/string/conversions.h>
 
 #include <stdlib.h>
+#include <shlobj_core.h>
 
 namespace AZ::Utils
 {
@@ -41,6 +43,14 @@ namespace AZ::Utils
             }
         }
 
+        wchar_t sysUserProfilePathW[MAX_PATH];
+        if (SUCCEEDED(SHGetFolderPath(0, CSIDL_PROFILE, 0, SHGFP_TYPE_DEFAULT, sysUserProfilePathW)))
+        {
+            AZ::IO::FixedMaxPathString sysUserProfilePathStr;
+            AZStd::to_string(sysUserProfilePathStr, sysUserProfilePathW);
+            return sysUserProfilePathStr;
+        }
+
         char userProfileBuffer[AZ::IO::MaxPathLength]{};
         size_t variableSize = 0;
         auto err = getenv_s(&variableSize, userProfileBuffer, AZ::IO::MaxPathLength, "USERPROFILE");

+ 41 - 64
Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.cpp

@@ -7,17 +7,18 @@
 */
 
 #include <AzToolsFramework/API/PythonLoader.h>
-#include <AzToolsFramework_Traits_Platform.h>
 #include <AzCore/Component/ComponentApplicationBus.h>
 #include <AzCore/IO/FileIO.h>
 #include <AzCore/IO/GenericStreams.h>
 #include <AzCore/IO/Path/Path.h>
+#include <AzCore/IO/SystemFile.h>
 #include <AzCore/Math/Sha1.h>
 #include <AzCore/std/containers/vector.h>
 #include <AzCore/std/string/string.h>
 #include <AzCore/std/string/string_view.h>
 #include <AzCore/std/string/conversions.h>
 #include <AzCore/std/string/tokenize.h>
+#include <AzCore/Serialization/Json/JsonUtils.h>
 #include <AzCore/Settings/ConfigParser.h>
 #include <AzCore/Utils/Utils.h>
 #include <AzFramework/IO/LocalFileIO.h>
@@ -26,82 +27,56 @@ namespace AzToolsFramework::EmbeddedPython
 {
     PythonLoader::PythonLoader()
     {
-        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-        // PYTHON_SHARED_LIBRARY_PATH must be defined in the build scripts and referencing the path to the python shared library
-        #if !defined(PYTHON_SHARED_LIBRARY_PATH)
-        #error "PYTHON_SHARED_LIBRARY_PATH is not defined"
-        #endif
-
-        // Construct the path to the shared python library within the venv folder
-        AZ::IO::FixedMaxPath engineRoot = AZ::IO::FixedMaxPath(AZ::Utils::GetEnginePath());
-        AZ::IO::FixedMaxPath thirdPartyRoot = PythonLoader::GetDefault3rdPartyPath(false);
-        AZ::IO::FixedMaxPath pythonVenvPath = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot);
-
-        AZ::IO::PathView libPythonName = AZ::IO::PathView(PYTHON_SHARED_LIBRARY_PATH).Filename();
-        AZ::IO::FixedMaxPath pythonVenvLibPath = pythonVenvPath / "lib" / libPythonName;
-
-        m_embeddedLibPythonModuleHandle = AZ::DynamicModuleHandle::Create(pythonVenvLibPath.StringAsPosix().c_str(), false);
-        bool loadResult = m_embeddedLibPythonModuleHandle->Load(false, true);
-        AZ_Error("PythonLoader", loadResult, "Failed to load %s.\n", libPythonName.StringAsPosix().c_str());
-        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-    }
-
-    PythonLoader::~PythonLoader()
-    {
-        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-        AZ_Assert(m_embeddedLibPythonModuleHandle, "DynamicModuleHandle for python was not created");
-        m_embeddedLibPythonModuleHandle->Unload();
-        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-    }
+        #if defined(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY)
 
-    AZ::IO::FixedMaxPath PythonLoader::GetDefault3rdPartyPath(bool createOnDemand)
-    {
-        AZ::IO::FixedMaxPath thirdPartyEnvPathPath;
-
-        // The highest priority for the 3rd party path is the environment variable 'LY_3RDPARTY_PATH'
-        static constexpr const char* env3rdPartyKey = "LY_3RDPARTY_PATH";
-        char env3rdPartyPath[AZ::IO::MaxPathLength] = { '\0' };
-        auto envOutcome = AZ::Utils::GetEnv(AZStd::span(env3rdPartyPath), env3rdPartyKey);
-        if (envOutcome && (strlen(env3rdPartyPath) > 0))
-        {
-            // If so, then use the path that is set as the third party path
-            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(env3rdPartyPath).LexicallyNormal();
-        }
-        // The next priority is to read the 3rd party directory from the manifest file
-        else if (auto manifest3rdPartyResult = AZ::Utils::Get3rdPartyDirectory(); manifest3rdPartyResult.IsSuccess())
+        // Determine if this is an sdk-engine build. For SDK engines, we want to prevent implicit python module loading.
+        [[maybe_unused]] bool isSdkEngine{ false };
+        auto engineSettingsPath = AZ::IO::FixedMaxPath{ AZ::Utils::GetEnginePath() } / "engine.json";
+        if (AZ::IO::SystemFile::Exists(engineSettingsPath.c_str()))
         {
-            thirdPartyEnvPathPath = manifest3rdPartyResult.GetValue();
+            auto loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(engineSettingsPath.c_str());
+            if (loadOutcome.IsSuccess())
+            {
+                auto& doc = loadOutcome.GetValue();
+                rapidjson::Value::MemberIterator sdkEngineFieldIter = doc.FindMember("sdk_engine");
+                if (sdkEngineFieldIter != doc.MemberEnd())
+                {
+                    isSdkEngine = sdkEngineFieldIter->value.GetBool();
+                }
+            }
         }
-        // Fallback to the default 3rd Party path based on the location of the manifest folder
-        else
+        if (!isSdkEngine)
         {
-            auto manifestPath = AZ::Utils::GetO3deManifestDirectory();
-            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(manifestPath) / "3rdParty";
+            m_embeddedLibPythonModuleHandle = AZ::DynamicModuleHandle::Create(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY, false);
+            bool loadResult = m_embeddedLibPythonModuleHandle->Load(false, true);
+            AZ_Error("PythonLoader", loadResult, "Failed to load " IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY "\n");
         }
+        #endif // IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY
+    }
 
-        if ((!AZ::IO::SystemFile::IsDirectory(thirdPartyEnvPathPath.c_str())) && createOnDemand)
+    PythonLoader::~PythonLoader()
+    {
+        #if defined(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY)
+        if (m_embeddedLibPythonModuleHandle)
         {
-            auto createPathResult = AZ::IO::SystemFile::CreateDir(thirdPartyEnvPathPath.c_str());
-            AZ_Assert(createPathResult, "Unable to create missing 3rd Party Folder '%s'", thirdPartyEnvPathPath.c_str())
+            m_embeddedLibPythonModuleHandle->Unload();
         }
-        return thirdPartyEnvPathPath;
+        #endif // IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonHomePath(AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonHomePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
-        AZ::IO::FixedMaxPath thirdPartyFolder = GetDefault3rdPartyPath(true);
-
         // The python HOME path relative to the executable depends on the host platform the package is created for
         #if AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
-        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot).ParentPath();
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(engineRoot, overridePythonBaseVenvPath).ParentPath();
         #else
-        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot);
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(engineRoot, overridePythonBaseVenvPath);
         #endif // AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
 
         return pythonHomePath;
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonVenvPath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
         // Perform the same hash calculation as cmake/CalculateEnginePathId.cmake
         /////
@@ -118,16 +93,18 @@ namespace AzToolsFramework::EmbeddedPython
         hasher.GetDigest(digest);
 
         // Construct the path to where the python venv based on the engine path should be located
-        AZ::IO::FixedMaxPath libPath = thirdPartyRoot;
+        AZ::IO::FixedMaxPath libPath = (overridePythonBaseVenvPath != nullptr) ? AZ::IO::FixedMaxPath(overridePythonBaseVenvPath) : 
+                                                                                 AZ::Utils::GetO3dePythonVenvRoot();
+
         // The ID is based on the first 32 bits of the digest, and formatted to at least 8-character wide hexadecimal representation
-        libPath /= AZ::IO::FixedMaxPathString::format("venv/%08x", digest[0]);
+        libPath /= AZ::IO::FixedMaxPathString::format("%08x", digest[0]);
         libPath = libPath.LexicallyNormal();
         return libPath;
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonExecutablePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
-        AZ::IO::FixedMaxPath pythonVenvConfig = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot) / "pyvenv.cfg";
+        AZ::IO::FixedMaxPath pythonVenvConfig = PythonLoader::GetPythonVenvPath(engineRoot, overridePythonBaseVenvPath) / "pyvenv.cfg";
         AZ::IO::SystemFileStream systemFileStream;
         if (!systemFileStream.Open(pythonVenvConfig.c_str(), AZ::IO::OpenMode::ModeRead))
         {
@@ -151,11 +128,11 @@ namespace AzToolsFramework::EmbeddedPython
         return AZ::IO::FixedMaxPath(pythonHome);
     }
 
-    void PythonLoader::ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor resultPathCallback)
+    void PythonLoader::ReadPythonEggLinkPaths(AZ::IO::PathView engineRoot, EggLinkPathVisitor resultPathCallback, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
         // Get the python venv path
         AZ::IO::FixedMaxPath pythonVenvSitePackages =
-            AZ::IO::FixedMaxPath(PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot)) / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
+            AZ::IO::FixedMaxPath(PythonLoader::GetPythonVenvPath(engineRoot, overridePythonBaseVenvPath)) / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
 
         // Always add the site-packages folder from the virtual environment into the path list
         resultPathCallback(pythonVenvSitePackages.LexicallyNormal());

+ 8 - 11
Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.h

@@ -26,35 +26,32 @@ namespace AzToolsFramework::EmbeddedPython
 
         //! Calculate the python home (PYTHONHOME) based on the engine root
         //! @param engineRoot The path to the engine root to locate the python home
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! @return The path of the python home path
-        static AZ::IO::FixedMaxPath GetPythonHomePath(AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonHomePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Collect the paths from all the egg-link files found in the python home
         //! paths used by the engine
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to locate the python home
         //! @param eggLinkPathVisitor The callback visitor function to receive the egg-link paths that are discovered
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         using EggLinkPathVisitor = AZStd::function<void(AZ::IO::PathView)>;
-        static void ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor eggLinkPathVisitor);
-
-        //! Get the default 3rd Party folder path.
-        //! @return The path of the 3rd Party root path
-        static AZ::IO::FixedMaxPath GetDefault3rdPartyPath(bool createOnDemand);
+        static void ReadPythonEggLinkPaths(AZ::IO::PathView engineRoot, EggLinkPathVisitor eggLinkPathVisitor, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Calculate the path to the engine's python virtual environment used for
         //! python home (PYTHONHOME) based on the engine root
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to locate the python venv path
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! @return The path of the python venv path
-        static AZ::IO::FixedMaxPath GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonVenvPath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Calculate the path to the where the python executable resides in. Note that this
         //! is not always the same path as the python home path
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! locate the python executable path
         //! @return The path of the python venv path
-        static AZ::IO::FixedMaxPath GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonExecutablePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
     private:
         AZStd::unique_ptr<AZ::DynamicModuleHandle> m_embeddedLibPythonModuleHandle;

+ 0 - 11
Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Linux.h

@@ -1,11 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      true
-#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      true

+ 0 - 10
Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Platform.h

@@ -1,10 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#include <AzToolsFramework_Traits_Linux.h>

+ 5 - 3
Code/Framework/AzToolsFramework/Platform/Linux/platform_linux.cmake

@@ -6,6 +6,8 @@
 #
 #
 
-get_target_property(libraries 3rdParty::Python INTERFACE_LINK_LIBRARIES)
-
-set(LY_COMPILE_DEFINITIONS PRIVATE PYTHON_SHARED_LIBRARY_PATH="${libraries}")
+set(LY_COMPILE_DEFINITIONS 
+    PRIVATE 
+        IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY="${LY_PYTHON_SHARED_LIB}"
+        AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
+)

+ 0 - 2
Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake

@@ -7,7 +7,5 @@
 #
 
 set(FILES
-    AzToolsFramework_Traits_Linux.h
-    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Linux.cpp
 )

+ 0 - 11
Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Mac.h

@@ -1,11 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      false
-#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      true

+ 0 - 10
Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework_Traits_Platform.h

@@ -1,10 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#include <AzToolsFramework_Traits_Mac.h>

+ 4 - 0
Code/Framework/AzToolsFramework/Platform/Mac/platform_mac.cmake

@@ -5,3 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0 OR MIT
 #
 #
+
+set(LY_COMPILE_DEFINITIONS 
+        PRIVATE
+            AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH)

+ 0 - 2
Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake

@@ -7,7 +7,5 @@
 #
 
 set(FILES
-    AzToolsFramework_Traits_Mac.h
-    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Mac.cpp
 )

+ 0 - 10
Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Platform.h

@@ -1,10 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#include <AzToolsFramework_Traits_Windows.h>

+ 0 - 11
Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework_Traits_Windows.h

@@ -1,11 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-#pragma once
-
-#define AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING      false
-#define AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH      false

+ 0 - 2
Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake

@@ -7,7 +7,5 @@
 #
 
 set(FILES
-    AzToolsFramework_Traits_Windows.h
-    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Windows.cpp
 )

+ 18 - 18
Code/Framework/AzToolsFramework/Tests/PythonLoaderTests.cpp

@@ -25,33 +25,32 @@ namespace UnitTest
 
         static constexpr AZStd::string_view s_testEnginePath = "O3de_path";
         static constexpr const char* s_testEnginePathHashId = "1af80774";
-        static constexpr const char* s_test3rdPartySubPath = ".o3de/3rdParty";
+        static constexpr const char* s_testPythonRootPath = ".o3de/3rdParty";
     };
 
     TEST_F(AzToolsFrameworkPythonLoaderFixture, TestGetPythonVenvPath_Valid)
     {
-        auto test3rdPartyPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
-        auto testVenvPath = test3rdPartyPath / "venv";
+        auto testPythonRootPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_testPythonRootPath;
+        auto testVenvRootPath = testPythonRootPath / "venv";
 
         AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
 
-        AZ::IO::SystemFile::CreateDir(test3rdPartyPath.String().c_str());
-        //AZ::IO::FileIOBase::GetInstance()->CreatePath(test3rdPartyPath.String().c_str());
+        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonVenvPath(engineRoot, testVenvRootPath.c_str());
 
-        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonVenvPath(test3rdPartyPath, engineRoot);
-
-        AZ::IO::FixedMaxPath expectedPath = testVenvPath / s_testEnginePathHashId;
+        AZ::IO::FixedMaxPath expectedPath = testVenvRootPath / s_testEnginePathHashId;
 
         EXPECT_TRUE(result == expectedPath);
     }
 
     TEST_F(AzToolsFrameworkPythonLoaderFixture, TestGetPythonVenvExecutablePath_Valid)
     {
+        auto testPythonRootPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_testPythonRootPath;
+        auto testVenvRootPath = testPythonRootPath / "venv";
+
         // Prepare the test venv pyvenv.cfg file in the expected location
-        AZ::IO::FixedMaxPath tempVenvRelativePath = AZ::IO::FixedMaxPath(s_test3rdPartySubPath) / "venv/" / s_testEnginePathHashId;
-        AZ::IO::FixedMaxPath tempVenvFullPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / tempVenvRelativePath;
-        AZ::IO::SystemFile::CreateDir(tempVenvFullPath.String().c_str());
-        AZ::IO::FixedMaxPath tempPyConfigFile = tempVenvRelativePath / "pyvenv.cfg";
+        AZ::IO::FixedMaxPath tempVenvPath = testVenvRootPath / s_testEnginePathHashId;
+        AZ::IO::SystemFile::CreateDir(tempVenvPath.c_str());
+        AZ::IO::FixedMaxPath tempPyConfigFile = tempVenvPath / "pyvenv.cfg";
         AZStd::string testPython3rdPartyPath = "/home/user/python/";
         AZStd::string testPyConfigFileContent = AZStd::string::format("home = %s\n"
                                                                       "include-system-site-packages = false\n"
@@ -62,9 +61,8 @@ namespace UnitTest
 
         // Test the method
         AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
-        AZ::IO::FixedMaxPath test3rdPartyRoot = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
 
-        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonExecutablePath(test3rdPartyRoot, engineRoot);
+        auto result = AzToolsFramework::EmbeddedPython::PythonLoader::GetPythonExecutablePath(engineRoot, testVenvRootPath.c_str());
         AZ::IO::FixedMaxPath expectedPath{ testPython3rdPartyPath };
 
         EXPECT_TRUE(result == expectedPath);
@@ -72,8 +70,11 @@ namespace UnitTest
 
     TEST_F(AzToolsFrameworkPythonLoaderFixture, TestReadPythonEggLinkPaths_Valid)
     {
+        auto testPythonRootPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_testPythonRootPath;
+        auto testVenvRootPath = testPythonRootPath / "venv";
+
         // Prepare the test folder and create dummy egg-link files
-        AZ::IO::FixedMaxPath testRelativeSiteLibsPath = AZ::IO::FixedMaxPath(s_test3rdPartySubPath) / "venv" / s_testEnginePathHashId / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
+        AZ::IO::FixedMaxPath testRelativeSiteLibsPath = testVenvRootPath / s_testEnginePathHashId / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
         AZ::IO::FixedMaxPath testFullSiteLIbsPath = m_tempDirectory.GetDirectoryAsFixedMaxPath() / testRelativeSiteLibsPath;
         AZ::IO::SystemFile::CreateDir(testFullSiteLIbsPath.String().c_str());
 
@@ -96,15 +97,14 @@ namespace UnitTest
 
         // Test the method
         AZ::IO::FixedMaxPath engineRoot{ s_testEnginePath };
-        AZ::IO::FixedMaxPath test3rdPartyRoot = m_tempDirectory.GetDirectoryAsFixedMaxPath() / s_test3rdPartySubPath;
         AZStd::vector<AZStd::string> resultEggLinkPaths;
         AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
-            test3rdPartyRoot,
             engineRoot,
             [&resultEggLinkPaths](AZ::IO::PathView path)
             {
                 resultEggLinkPaths.emplace_back(path.Native());
-            } );
+            },
+            testVenvRootPath.c_str());
 
         // Sort the expected and actual lists
         AZStd::sort(expectedResults.begin(), expectedResults.end());

+ 1 - 3
Code/Tools/ProjectManager/Source/PythonBindings.cpp

@@ -318,11 +318,9 @@ namespace O3DE::ProjectManager
             // alters the behavior of the initializer to not compute default search paths. See
             // https://docs.python.org/3/c-api/init.html#c.Py_SetPath
 
-            AZ::IO::FixedMaxPath thirdPartyFolder = AzToolsFramework::EmbeddedPython::PythonLoader::GetDefault3rdPartyPath(false);
-
             AZStd::vector<AZ::IO::Path> extendedPaths;
             AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
-                thirdPartyFolder, m_enginePath.c_str(), [&extendedPaths](AZ::IO::PathView path)
+                m_enginePath.c_str(), [&extendedPaths](AZ::IO::PathView path)
                 {
                     extendedPaths.emplace_back(path);
                 });

+ 0 - 3
Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp

@@ -496,10 +496,7 @@ namespace EditorPythonBindings
         //   5 - user(dev)
 
         // 1 - The python venv site-packages
-        AZ::IO::FixedMaxPath thirdPartyFolder = AzToolsFramework::EmbeddedPython::PythonLoader::GetDefault3rdPartyPath(false);
-
         AzToolsFramework::EmbeddedPython::PythonLoader::ReadPythonEggLinkPaths(
-            thirdPartyFolder,
             AZ::Utils::GetEnginePath().c_str(),
             [&pythonPathStack](AZ::IO::PathView path)
             {

+ 0 - 7
cmake/3rdParty/Platform/Linux/BuiltInPackages_linux_aarch64.cmake

@@ -46,10 +46,3 @@ ly_associate_package(PACKAGE_NAME pyside2-5.15.2.1-py3.10-rev4-linux-aarch64
 ly_associate_package(PACKAGE_NAME SQLite-3.37.2-rev1-linux-aarch64                           TARGETS SQLite                      PACKAGE_HASH 5cc1fd9294af72514eba60509414e58f1a268996940be31d0ab6919383f05118)
 ly_associate_package(PACKAGE_NAME AwsIotDeviceSdkCpp-1.15.2-rev1-linux-aarch64               TARGETS AwsIotDeviceSdkCpp          PACKAGE_HASH 0bac80fc09094c4fd89a845af57ebe4ef86ff8d46e92a448c6986f9880f9ee62)
 ly_associate_package(PACKAGE_NAME vulkan-validationlayers-1.2.198-rev1-linux-aarch64         TARGETS vulkan-validationlayers     PACKAGE_HASH e67a15a95e14397ccdffd70d17f61079e5720fea22b0d21e135497312419a23f)
-
-set(AZ_USE_PHYSX5 OFF CACHE BOOL "When ON PhysX Gem will use PhysX 5 SDK")
-if(AZ_USE_PHYSX5)
-    ly_associate_package(PACKAGE_NAME PhysX-5.1.1-rev1-linux-aarch64                         TARGETS PhysX                       PACKAGE_HASH ba14e96fc4b85d5ef5dcf8c8ee58fd37ab92cfef790872d67c9a389aeca24c19)
-else()
-    ly_associate_package(PACKAGE_NAME PhysX-4.1.2.29882248-rev5-linux-aarch64                TARGETS PhysX                       PACKAGE_HASH 7fa00d7d4f7532cf068246d4e424ce319529dfa09fb759d251676f2c59f6d50c)
-endif()

+ 0 - 7
cmake/3rdParty/Platform/Linux/BuiltInPackages_linux_x86_64.cmake

@@ -46,10 +46,3 @@ ly_associate_package(PACKAGE_NAME pyside2-5.15.2.1-py3.10-rev6-linux
 ly_associate_package(PACKAGE_NAME SQLite-3.37.2-rev1-linux                          TARGETS SQLite                      PACKAGE_HASH bee80d6c6db3e312c1f4f089c90894436ea9c9b74d67256d8c1fb00d4d81fe46)
 ly_associate_package(PACKAGE_NAME AwsIotDeviceSdkCpp-1.15.2-rev1-linux              TARGETS AwsIotDeviceSdkCpp          PACKAGE_HASH 83fc1711404d3e5b2faabb1134e97cc92b748d8b87ff4ea99599d8c750b8eff0)
 ly_associate_package(PACKAGE_NAME vulkan-validationlayers-1.2.198-rev1-linux        TARGETS vulkan-validationlayers     PACKAGE_HASH 9195c7959695bcbcd1bc1dc5c425c14639a759733b3abe2ffa87eb3915b12c71)
-
-set(AZ_USE_PHYSX5 OFF CACHE BOOL "When ON PhysX Gem will use PhysX 5 SDK")
-if(AZ_USE_PHYSX5)
-    ly_associate_package(PACKAGE_NAME PhysX-5.1.1-rev1-linux                            TARGETS PhysX                       PACKAGE_HASH 343e3123882f53748c938c41f4d1c9ab6ec171e4f970ccd9263c5ab191110410)
-else()
-    ly_associate_package(PACKAGE_NAME PhysX-4.1.2.29882248-rev6-linux                   TARGETS PhysX                       PACKAGE_HASH 220a420d0e48b27d0dfe4ef71774a40cd6015938a58f8870d9e666d7c1537c94)
-endif()

+ 12 - 1
cmake/3rdParty/Platform/Linux/Python_linux_aarch64.cmake

@@ -18,6 +18,7 @@ ly_set(LY_PYTHON_PACKAGE_HASH 30bc2731e2ac54d8e22d36ab15e30b77aefe2dce146ef92d6f
 ly_set(LY_PYTHON_BIN_PATH "python/bin")
 ly_set(LY_PYTHON_LIB_PATH "python/lib")
 ly_set(LY_PYTHON_EXECUTABLE "python")
+ly_set(LY_PYTHON_SHARED_LIB "libpython3.10.so.1.0")
 
 # Python venv relative paths
 ly_set(LY_PYTHON_VENV_BIN_PATH "bin")
@@ -25,4 +26,14 @@ ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
 ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
 ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
 
-ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
+function(ly_post_python_venv_install venv_path)
+
+    # We need to create a symlink to the shared library in the venv
+    execute_process(COMMAND ln -s -f "${PYTHON_PACKAGES_ROOT_PATH}/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_LIB_PATH}/${LY_PYTHON_SHARED_LIB}" ${LY_PYTHON_SHARED_LIB}
+                    WORKING_DIRECTORY "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_LIB_PATH}"
+                    RESULT_VARIABLE command_result)
+    if (NOT ${command_result} EQUAL 0)
+        message(WARNING "Unable to create a venv shared library link.")
+    endif()
+
+endfunction()

+ 12 - 1
cmake/3rdParty/Platform/Linux/Python_linux_x86_64.cmake

@@ -18,6 +18,7 @@ ly_set(LY_PYTHON_PACKAGE_HASH a7832f9170a3ac93fbe678e9b3d99a977daa03bb667d258859
 ly_set(LY_PYTHON_BIN_PATH "python/bin")
 ly_set(LY_PYTHON_LIB_PATH "python/lib")
 ly_set(LY_PYTHON_EXECUTABLE "python")
+ly_set(LY_PYTHON_SHARED_LIB "libpython3.10.so.1.0")
 
 # Python venv relative paths
 ly_set(LY_PYTHON_VENV_BIN_PATH "bin")
@@ -25,4 +26,14 @@ ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
 ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
 ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
 
-ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
+function(ly_post_python_venv_install venv_path)
+
+    # We need to create a symlink to the shared library in the venv
+    execute_process(COMMAND ln -s -f "${PYTHON_PACKAGES_ROOT_PATH}/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_LIB_PATH}/${LY_PYTHON_SHARED_LIB}" ${LY_PYTHON_SHARED_LIB}
+                    WORKING_DIRECTORY "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_LIB_PATH}"
+                    RESULT_VARIABLE command_result)
+    if (NOT ${command_result} EQUAL 0)
+        message(WARNING "Unable to create a venv shared library link.")
+    endif()
+
+endfunction()

+ 0 - 7
cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake

@@ -45,10 +45,3 @@ ly_associate_package(PACKAGE_NAME lz4-1.9.4-rev1-mac
 ly_associate_package(PACKAGE_NAME azslc-1.8.19-rev1-mac                             TARGETS azslc                       PACKAGE_HASH f3ca6bb1dc44ffe3470aa589bbbbb39cc4c2e7faa8c9f7760388aa41f06b4cce)
 ly_associate_package(PACKAGE_NAME SQLite-3.37.2-rev2-mac                            TARGETS SQLite                      PACKAGE_HASH b7d9abdb68045003e030e1a9a805db1aefa5e8fde6dccfbb4fab3a06249a41fc)
 ly_associate_package(PACKAGE_NAME AwsIotDeviceSdkCpp-1.15.2-rev2-mac                TARGETS AwsIotDeviceSdkCpp          PACKAGE_HASH 4854edb7b88fa6437b4e69e87d0ee111a25313ac2a2db5bb2f8b674ba0974f95)
-
-set(AZ_USE_PHYSX5 OFF CACHE BOOL "When ON PhysX Gem will use PhysX 5 SDK")
-if(AZ_USE_PHYSX5)
-    ly_associate_package(PACKAGE_NAME PhysX-5.1.1-rev1-mac                              TARGETS PhysX                       PACKAGE_HASH 1c227e795f43bd0e4f27d352b2d8f75d1e089ef5ba6bf472f478a1169b71db4d)
-else()
-    ly_associate_package(PACKAGE_NAME PhysX-4.1.2.29882248-rev5-mac                     TARGETS PhysX                       PACKAGE_HASH 83940b3876115db82cd8ffcb9e902278e75846d6ad94a41e135b155cee1ee186)
-endif()

+ 3 - 1
cmake/3rdParty/Platform/Mac/Python_mac.cmake

@@ -25,4 +25,6 @@ ly_set(LY_PYTHON_VENV_LIB_PATH "lib")
 ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/python3.10/site-packages")
 ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python")
 
-ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
+function(ly_post_python_venv_install venv_path)
+    # Noop
+endfunction()

+ 3 - 1
cmake/3rdParty/Platform/Windows/Python_windows.cmake

@@ -25,4 +25,6 @@ ly_set(LY_PYTHON_VENV_LIB_PATH "Lib")
 ly_set(LY_PYTHON_VENV_SITE_PACKAGES "${LY_PYTHON_VENV_LIB_PATH}/site-packages")
 ly_set(LY_PYTHON_VENV_PYTHON "${LY_PYTHON_VENV_BIN_PATH}/python.exe")
 
-ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
+function(ly_post_python_venv_install venv_path)
+    # Noop
+endfunction()

+ 29 - 4
cmake/3rdPartyPackages.cmake

@@ -251,8 +251,11 @@ function(ly_package_internal_download_package package_name url_variable)
 
     foreach(server_url ${LY_PACKAGE_SERVER_URLS})
         set(download_url ${server_url}/${package_name}${LY_PACKAGE_EXT})
-        set(download_target ${LY_PACKAGE_DOWNLOAD_CACHE_LOCATION}/${package_name}${LY_PACKAGE_EXT})
-        
+
+        ly_package_get_target_cache(${package_name} package_download_cache_location)
+
+        set(download_target ${package_download_cache_location}/${package_name}${LY_PACKAGE_EXT})
+
         file(REMOVE ${download_target})
 
         ly_package_message(STATUS "ly_package: trying to download ${download_url} to ${download_target}")
@@ -389,6 +392,16 @@ function(ly_package_get_target_folder package_name output_variable_name)
     endif()
 endfunction()
 
+#! Get the target cache folder
+function(ly_package_get_target_cache package_name output_variable_name)
+    get_property(overridden_location GLOBAL PROPERTY LY_PACKAGE_DOWNLOAD_CACHE_LOCATION_${package_name})
+    if (overridden_location)
+        set(${output_variable_name} ${overridden_location} PARENT_SCOPE)
+    else()
+        set(${output_variable_name} ${LY_PACKAGE_DOWNLOAD_CACHE_LOCATION} PARENT_SCOPE)
+    endif()
+endfunction()
+
 #! given the name of a package, validate that all files are present and match as appropriate
 function(ly_validate_package package_name)
     ly_package_message(STATUS "ly_package: Validating ${package_name}...")
@@ -427,7 +440,8 @@ function(ly_validate_package package_name)
         # The package hash is always checked automatically on first downloard regardless of the value of this
         # variable, so if this variable is true, a user explicitly asked to do this.
         message(STATUS "Checking downloaded package ${package_name} because LY_PACKAGE_VALIDATE_PACKAGE is TRUE")
-        set(temp_download_target ${LY_PACKAGE_DOWNLOAD_CACHE_LOCATION}/${package_name}${LY_PACKAGE_EXT})
+        ly_package_get_target_cache(${package_name} package_download_cache_location)
+        set(temp_download_target ${package_download_cache_location}/${package_name}${LY_PACKAGE_EXT})
         ly_get_package_expected_hash(${package_name} expected_package_hash)
         if (EXISTS ${temp_download_target})
             file(SHA256 ${temp_download_target} existing_hash)
@@ -479,7 +493,8 @@ function(ly_force_download_package package_name)
     set(final_folder ${DOWNLOAD_LOCATION}/${package_name})
 
     # is the package already present in the download cache, with the correct hash?
-    set(temp_download_target ${LY_PACKAGE_DOWNLOAD_CACHE_LOCATION}/${package_name}${LY_PACKAGE_EXT})
+    ly_package_get_target_cache(${package_name} package_download_cache_location)
+    set(temp_download_target ${package_download_cache_location}/${package_name}${LY_PACKAGE_EXT})
     ly_get_package_expected_hash(${package_name} expected_package_hash)
 
     # can we reuse the download we already have in our download cache?
@@ -679,6 +694,16 @@ macro(ly_set_package_download_location package_name download_location)
     set_property(GLOBAL PROPERTY LY_PACKAGE_DOWNLOAD_LOCATION_${package_name} ${download_location})
 endmacro()
 
+# ly_set_package_download_cache_location - OPTIONAL.
+# Similar to ly_set_package_download_location, this set the download 
+# cache location for a particular package.
+# note that package_name is expected to be the actual package name, not 
+# the find_package(...)  name!
+macro(ly_set_package_download_cache_location package_name download_location)
+    set_property(GLOBAL PROPERTY LY_PACKAGE_DOWNLOAD_CACHE_LOCATION_${package_name} ${download_location})
+endmacro()
+
+
 # ly_download_associated_package  - main public function
 # this just checks to see if the find_library_name (like 'zlib', not a package name)
 # is associated with a package, as above.  If it is, it makes sure that the package

+ 20 - 24
cmake/LYPython.cmake

@@ -34,20 +34,23 @@ execute_process(COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Ca
                 OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 # Normalize the expected location of the Python venv and set its path globally
-set(PYTHON_VENV_PATH "${LY_3RDPARTY_PATH}/venv/${ENGINE_SOURCE_PATH_ID}")
-cmake_path(NORMAL_PATH PYTHON_VENV_PATH )
-ly_set(LY_PYTHON_VENV_PATH ${PYTHON_VENV_PATH})
-
-# On Linux systems, we need to create a symlink to the shared library as well
-# due to the fact that a symlink is created for the python executable inside 
-# the virtual environment, but the python executable is custom built to use
-# an RPATH set to $ORIGIN/../lib to match the install structure in the package.
-if ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux")
-    set(LY_LINK_TO_SHARED_LIBRARY TRUE)
+# The root Python folder is based off of the same folder as the manifest file
+if(DEFINED ENV{USERPROFILE} AND EXISTS $ENV{USERPROFILE})
+    set(PYTHON_ROOT_PATH "$ENV{USERPROFILE}/.o3de/Python") # Windows
 else()
-    set(LY_LINK_TO_SHARED_LIBRARY FALSE)
+    set(PYTHON_ROOT_PATH "$ENV{HOME}/.o3de/Python") # Unix
 endif()
 
+set(PYTHON_PACKAGES_ROOT_PATH "${PYTHON_ROOT_PATH}/packages")
+cmake_path(NORMAL_PATH PYTHON_PACKAGES_ROOT_PATH )
+
+set(PYTHON_PACKAGE_CACHE_ROOT_PATH "${PYTHON_ROOT_PATH}/downloaded_packages")
+cmake_path(NORMAL_PATH PYTHON_PACKAGE_CACHE_ROOT_PATH )
+
+set(PYTHON_VENV_PATH "${PYTHON_ROOT_PATH}/venv/${ENGINE_SOURCE_PATH_ID}")
+cmake_path(NORMAL_PATH PYTHON_VENV_PATH )
+
+ly_set(LY_PYTHON_VENV_PATH ${PYTHON_VENV_PATH})
 
 function(ly_setup_python_venv)
 
@@ -68,6 +71,7 @@ function(ly_setup_python_venv)
         else()
             # Sanity check to make sure the python launcher in the venv works
             execute_process(COMMAND ${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_PYTHON} --version
+                            OUTPUT_QUIET
                             RESULT_VARIABLE command_result)
             if (NOT ${command_result} EQUAL 0)
                 message(STATUS "Error validating python inside the venv. Reinstalling")
@@ -86,8 +90,8 @@ function(ly_setup_python_venv)
         # a link to the shared library within the created virtual environment before proceeding with
         # the pip install command.
         message(STATUS "Creating Python venv at ${PYTHON_VENV_PATH}")
-        execute_process(COMMAND "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}/${LY_PYTHON_EXECUTABLE}" -m venv "${PYTHON_VENV_PATH}" --without-pip --clear
-                        WORKING_DIRECTORY "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}"
+        execute_process(COMMAND "${PYTHON_PACKAGES_ROOT_PATH}/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}/${LY_PYTHON_EXECUTABLE}" -m venv "${PYTHON_VENV_PATH}" --without-pip --clear
+                        WORKING_DIRECTORY "${PYTHON_PACKAGES_ROOT_PATH}/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_BIN_PATH}"
                         COMMAND_ECHO STDOUT
                         RESULT_VARIABLE command_result)
 
@@ -95,17 +99,7 @@ function(ly_setup_python_venv)
             message(FATAL_ERROR "Error creating a venv")
         endif()
 
-        if (LY_LINK_TO_SHARED_LIBRARY)
-            # Create a symlink to the package's python shared library inside the virtual environments
-            # own lib sub folder to match the original package structure
-            execute_process(COMMAND ln -s -f "${LY_3RDPARTY_PATH}/packages/${LY_PYTHON_PACKAGE_NAME}/${LY_PYTHON_LIB_PATH}/libpython3.10.so.1.0" libpython3.10.so.1.0
-                            WORKING_DIRECTORY "${PYTHON_VENV_PATH}/${LY_PYTHON_VENV_LIB_PATH}"
-                            COMMAND_ECHO STDOUT
-                            RESULT_VARIABLE command_result)
-            if (NOT ${command_result} EQUAL 0)
-                message(WARNING "Unable to createa venv shared library link.")
-            endif()
-        endif()
+        ly_post_python_venv_install(${PYTHON_VENV_PATH})
 
         # Manually install pip into the virtual environment
         message(STATUS "Installing pip into venv at ${PYTHON_VENV_PATH} (${PYTHON_VENV_PATH}/${LY_PYTHON_BIN_PATH})")
@@ -307,6 +301,8 @@ endfunction()
 
 # We need to download the associated Python package early and install the venv 
 ly_associate_package(PACKAGE_NAME ${LY_PYTHON_PACKAGE_NAME} TARGETS "Python" PACKAGE_HASH ${LY_PYTHON_PACKAGE_HASH})
+ly_set_package_download_location(${LY_PYTHON_PACKAGE_NAME} ${PYTHON_PACKAGES_ROOT_PATH})
+ly_set_package_download_cache_location(${LY_PYTHON_PACKAGE_NAME} ${PYTHON_PACKAGE_CACHE_ROOT_PATH})
 ly_download_associated_package(Python)
 ly_setup_python_venv()
 

+ 11 - 0
cmake/Platform/Common/RuntimeDependencies_common.cmake

@@ -499,6 +499,17 @@ function(ly_delayed_generate_runtime_dependencies)
             unset(target_file)
         endif()
 
+        if(DEFINED ENV{USERPROFILE} AND EXISTS $ENV{USERPROFILE})
+            set(PYTHON_ROOT_PATH "$ENV{USERPROFILE}/.o3de/Python") # Windows
+        else()
+            set(PYTHON_ROOT_PATH "$ENV{HOME}/.o3de/Python") # Unix
+        endif()
+        set(PYTHON_PACKAGES_ROOT_PATH "${PYTHON_ROOT_PATH}/packages")
+        cmake_path(NORMAL_PATH PYTHON_PACKAGES_ROOT_PATH )
+
+        set(LY_CURRENT_PYTHON_PACKAGE_PATH "${PYTHON_PACKAGES_ROOT_PATH}/${LY_PYTHON_PACKAGE_NAME}")
+        cmake_path(NORMAL_PATH LY_CURRENT_PYTHON_PACKAGE_PATH )
+
         ly_file_read(${LY_RUNTIME_DEPENDENCIES_TEMPLATE} template_file)
         string(CONFIGURE "${LY_COPY_COMMANDS}" LY_COPY_COMMANDS @ONLY)
         string(CONFIGURE "${template_file}" configured_template_file @ONLY)

+ 13 - 1
cmake/Platform/Linux/runtime_dependencies_linux.cmake.in

@@ -33,7 +33,18 @@ function(ly_copy source_files relative_target_directory)
         cmake_PATH(GET source_file EXTENSION target_filename_ext)
         cmake_path(APPEND target_file "${target_directory}" "${target_filename}")
         cmake_path(COMPARE "${source_file}" EQUAL "${target_file}" same_location)
-        if(NOT ${same_location})
+
+        if("${source_file}" MATCHES "@LY_PYTHON_SHARED_LIB@")
+            # Special case: Since the shared python library is loaded dynamically, it has the potential of being loaded from multiple
+            # sources. To make sure that duplicate loads to the shared python library load to the same handle, create a soft-symlink
+            # instead so that the dlopen call can resolve the library properly
+
+            # Use the relative path to the package source for the link
+            set(source_python_package_abs @LY_CURRENT_PYTHON_PACKAGE_PATH@/python/lib/@LY_PYTHON_SHARED_LIB@)
+            execute_process(COMMAND ln -s -f "${source_python_package_abs}" @LY_PYTHON_SHARED_LIB@
+                            WORKING_DIRECTORY "${target_directory}")
+
+        elseif(NOT ${same_location})
             file(LOCK "${target_file}.lock" GUARD FUNCTION TIMEOUT 300)
             file(SIZE "${source_file}" source_file_size)
             if(EXISTS "${target_file}")
@@ -41,6 +52,7 @@ function(ly_copy source_files relative_target_directory)
             else()
                 set(target_file_size 0)
             endif()
+
             if((NOT source_file_size EQUAL target_file_size) OR "${source_file}" IS_NEWER_THAN "${target_file}")
                 message(STATUS "Copying \"${source_file}\" to \"${target_directory}\"...")
                 file(MAKE_DIRECTORY "${full_target_directory}")

+ 0 - 2
python/.gitignore

@@ -1,2 +0,0 @@
-downloaded_packages/
-runtime/

+ 0 - 1
python/.p4ignore

@@ -1 +0,0 @@
-runtime/

+ 1 - 1
python/python.cmd

@@ -40,7 +40,7 @@ IF "" == "%LY_3RDPARTY_PATH%" (
     SET LY_3RDPARTY_PATH=%USERPROFILE%\.o3de\3rdParty
 )
 
-SET PYTHON_VENV=%LY_3RDPARTY_PATH%\venv\%ENGINE_ID%
+SET PYTHON_VENV=%USERPROFILE%\.o3de\Python\venv\%ENGINE_ID%
 SET PYTHON_VENV_ACTIVATE=%PYTHON_VENV%\Scripts\activate.bat
 SET PYTHON_VENV_DEACTIVATE=%PYTHON_VENV%\Scripts\deactivate.bat
 IF [%1] EQU [debug] (

+ 2 - 7
python/python.sh

@@ -52,14 +52,9 @@ then
     exit 1
 fi
 
-if [ "$LY_3RDPARTY_PATH" == "" ]
-then
-    LY_3RDPARTY_PATH=$HOME/.o3de/3rdParty
-fi
-
 # Set the expected location of the python venv for this engine and the locations of the critical scripts/executables 
 # needed to run python within the venv properly
-PYTHON_VENV=$LY_3RDPARTY_PATH/venv/$ENGINE_ID
+PYTHON_VENV=$HOME/.o3de/Python/venv/$ENGINE_ID
 PYTHON_VENV_ACTIVATE=$PYTHON_VENV/bin/activate
 PYTHON_VENV_PYTHON=$PYTHON_VENV/bin/python
 if [ ! -f $PYTHON_VENV_PYTHON ]
@@ -96,5 +91,5 @@ source $PYTHON_VENV_ACTIVATE
 # is the one that is loaded by injecting the shared lib path before invoking python. Otherwise,
 # the shared library may not be found or it could load it from a different location.
 PYTHON_LIB_PATH=$PYTHON_VENV/lib
-PYTHONNOUSERSITE=1 LD_LIBRARY_PATH="$PYTHON_LIB_PATH:$LD_LIBRARY_PATH" "$PYTHON_VENV_PYTHON" "$@"
+PYTHONNOUSERSITE=1 LD_LIBRARY_PATH="$PYTHON_LIB_PATH:$LD_LIBRARY_PATH" PYTHONPATH= "$PYTHON_VENV_PYTHON" "$@"
 exit $?

+ 1 - 1
scripts/build/ci_build.py

@@ -88,7 +88,7 @@ def build(build_config_filename, build_platform, build_type):
                 env_params[v] = build_params[v]
             print('    {} = {} {}'.format(v, env_params[v], '(environment override)' if existing_param else ''))
     print('--------------------------------------------------------------------------------', flush=True)
-    process_return = subprocess.run([build_cmd_path], cwd=cwd_dir, env=env_params)
+    process_return = subprocess.run([build_cmd_path], cwd=cwd_dir, env=env_params, shell=True)
     print('--------------------------------------------------------------------------------')
     if process_return.returncode != 0:
         print('[ci_build] FAIL: Command {} returned {}'.format(build_cmd_path, process_return.returncode), flush=True)