PythonSystemComponent.cpp 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756
  1. /*
  2. * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
  3. * its licensors.
  4. *
  5. * For complete copyright and license terms please see the LICENSE at the root of this
  6. * distribution (the "License"). All use of this software is governed by the License,
  7. * or, if provided, by the license below or the license accompanying this file. Do not
  8. * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
  9. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. *
  11. */
  12. #include <PythonSystemComponent.h>
  13. #include <EditorPythonBindings/EditorPythonBindingsBus.h>
  14. #include <Source/PythonCommon.h>
  15. #include <pybind11/pybind11.h>
  16. #include <pybind11/embed.h>
  17. #include <pybind11/eval.h>
  18. #include <osdefs.h> // for DELIM
  19. #include <AzCore/Component/EntityId.h>
  20. #include <AzCore/IO/SystemFile.h>
  21. #include <AzCore/Module/DynamicModuleHandle.h>
  22. #include <AzCore/Module/Module.h>
  23. #include <AzCore/Module/ModuleManagerBus.h>
  24. #include <AzCore/PlatformDef.h>
  25. #include <AzCore/Serialization/EditContext.h>
  26. #include <AzCore/Serialization/SerializeContext.h>
  27. #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
  28. #include <AzCore/std/string/conversions.h>
  29. #include <AzCore/StringFunc/StringFunc.h>
  30. #include <AzCore/Utils/Utils.h>
  31. #include <AzFramework/API/ApplicationAPI.h>
  32. #include <AzFramework/Asset/AssetSystemComponent.h>
  33. #include <AzFramework/IO/LocalFileIO.h>
  34. #include <AzFramework/CommandLine/CommandRegistrationBus.h>
  35. #include <AzFramework/StringFunc/StringFunc.h>
  36. #include <AzToolsFramework/API/EditorPythonConsoleBus.h>
  37. namespace Platform
  38. {
  39. // Implemented in each different platform's implentation files, as it differs per platform.
  40. bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot);
  41. AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot);
  42. }
  43. // this is called the first time a Python script contains "import azlmbr"
  44. PYBIND11_EMBEDDED_MODULE(azlmbr, m)
  45. {
  46. EditorPythonBindings::EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindings::EditorPythonBindingsNotificationBus::Events::OnImportModule, m.ptr());
  47. }
  48. namespace RedirectOutput
  49. {
  50. using RedirectOutputFunc = AZStd::function<void(const char*)>;
  51. struct RedirectOutput
  52. {
  53. PyObject_HEAD
  54. RedirectOutputFunc write;
  55. };
  56. PyObject* RedirectWrite(PyObject* self, PyObject* args)
  57. {
  58. std::size_t written(0);
  59. RedirectOutput* selfimpl = reinterpret_cast<RedirectOutput*>(self);
  60. if (selfimpl->write)
  61. {
  62. char* data;
  63. if (!PyArg_ParseTuple(args, "s", &data))
  64. {
  65. return PyLong_FromSize_t(0);
  66. }
  67. selfimpl->write(data);
  68. written = strlen(data);
  69. }
  70. return PyLong_FromSize_t(written);
  71. }
  72. PyObject* RedirectFlush([[maybe_unused]] PyObject* self, [[maybe_unused]] PyObject* args)
  73. {
  74. // no-op
  75. return Py_BuildValue("");
  76. }
  77. PyMethodDef RedirectMethods[] =
  78. {
  79. {"write", RedirectWrite, METH_VARARGS, "sys.stdout.write"},
  80. {"flush", RedirectFlush, METH_VARARGS, "sys.stdout.flush"},
  81. {"write", RedirectWrite, METH_VARARGS, "sys.stderr.write"},
  82. {"flush", RedirectFlush, METH_VARARGS, "sys.stderr.flush"},
  83. {0, 0, 0, 0} // sentinel
  84. };
  85. PyTypeObject RedirectOutputType =
  86. {
  87. PyVarObject_HEAD_INIT(0, 0)
  88. "azlmbr_redirect.RedirectOutputType", // tp_name
  89. sizeof(RedirectOutput), /* tp_basicsize */
  90. 0, /* tp_itemsize */
  91. 0, /* tp_dealloc */
  92. 0, /* tp_print */
  93. 0, /* tp_getattr */
  94. 0, /* tp_setattr */
  95. 0, /* tp_reserved */
  96. 0, /* tp_repr */
  97. 0, /* tp_as_number */
  98. 0, /* tp_as_sequence */
  99. 0, /* tp_as_mapping */
  100. 0, /* tp_hash */
  101. 0, /* tp_call */
  102. 0, /* tp_str */
  103. 0, /* tp_getattro */
  104. 0, /* tp_setattro */
  105. 0, /* tp_as_buffer */
  106. Py_TPFLAGS_DEFAULT, /* tp_flags */
  107. "azlmbr_redirect objects", /* tp_doc */
  108. 0, /* tp_traverse */
  109. 0, /* tp_clear */
  110. 0, /* tp_richcompare */
  111. 0, /* tp_weaklistoffset */
  112. 0, /* tp_iter */
  113. 0, /* tp_iternext */
  114. RedirectMethods, /* tp_methods */
  115. 0, /* tp_members */
  116. 0, /* tp_getset */
  117. 0, /* tp_base */
  118. 0, /* tp_dict */
  119. 0, /* tp_descr_get */
  120. 0, /* tp_descr_set */
  121. 0, /* tp_dictoffset */
  122. 0, /* tp_init */
  123. 0, /* tp_alloc */
  124. 0 /* tp_new */
  125. };
  126. PyModuleDef RedirectOutputModule = { PyModuleDef_HEAD_INIT, "azlmbr_redirect", 0, -1, 0, };
  127. // Internal state
  128. PyObject* g_redirect_stdout = nullptr;
  129. PyObject* g_redirect_stdout_saved = nullptr;
  130. PyObject* g_redirect_stderr = nullptr;
  131. PyObject* g_redirect_stderr_saved = nullptr;
  132. PyMODINIT_FUNC PyInit_RedirectOutput(void)
  133. {
  134. g_redirect_stdout = nullptr;
  135. g_redirect_stdout_saved = nullptr;
  136. g_redirect_stderr = nullptr;
  137. g_redirect_stderr_saved = nullptr;
  138. RedirectOutputType.tp_new = PyType_GenericNew;
  139. if (PyType_Ready(&RedirectOutputType) < 0)
  140. {
  141. return 0;
  142. }
  143. PyObject* m = PyModule_Create(&RedirectOutputModule);
  144. if (m)
  145. {
  146. Py_INCREF(&RedirectOutputType);
  147. PyModule_AddObject(m, "Redirect", reinterpret_cast<PyObject*>(&RedirectOutputType));
  148. }
  149. return m;
  150. }
  151. void SetRedirection(const char* funcname, PyObject*& saved, PyObject*& current, RedirectOutputFunc func)
  152. {
  153. if (PyType_Ready(&RedirectOutputType) < 0)
  154. {
  155. AZ_Warning("python", false, "RedirectOutputType not ready!");
  156. return;
  157. }
  158. if (!current)
  159. {
  160. saved = PySys_GetObject(funcname); // borrowed
  161. current = RedirectOutputType.tp_new(&RedirectOutputType, 0, 0);
  162. }
  163. RedirectOutput* redirectOutput = reinterpret_cast<RedirectOutput*>(current);
  164. redirectOutput->write = func;
  165. PySys_SetObject(funcname, current);
  166. }
  167. void ResetRedirection(const char* funcname, PyObject*& saved, PyObject*& current)
  168. {
  169. if (current)
  170. {
  171. PySys_SetObject(funcname, saved);
  172. }
  173. Py_XDECREF(current);
  174. current = nullptr;
  175. }
  176. PyObject* s_RedirectModule = nullptr;
  177. void Intialize(PyObject* module)
  178. {
  179. using namespace AzToolsFramework;
  180. s_RedirectModule = module;
  181. SetRedirection("stdout", g_redirect_stdout_saved, g_redirect_stdout, [](const char* msg)
  182. {
  183. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnTraceMessage, msg);
  184. });
  185. SetRedirection("stderr", g_redirect_stderr_saved, g_redirect_stderr, [](const char* msg)
  186. {
  187. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnErrorMessage, msg);
  188. });
  189. PySys_WriteStdout("RedirectOutput installed");
  190. }
  191. void Shutdown()
  192. {
  193. ResetRedirection("stdout", g_redirect_stdout_saved, g_redirect_stdout);
  194. ResetRedirection("stderr", g_redirect_stderr_saved, g_redirect_stderr);
  195. Py_XDECREF(s_RedirectModule);
  196. s_RedirectModule = nullptr;
  197. }
  198. } // namespace RedirectOutput
  199. namespace EditorPythonBindings
  200. {
  201. void PythonSystemComponent::Reflect(AZ::ReflectContext* context)
  202. {
  203. if (AZ::SerializeContext* serialize = azrtti_cast<AZ::SerializeContext*>(context))
  204. {
  205. serialize->Class<PythonSystemComponent, AZ::Component>()
  206. ->Version(1)
  207. ->Attribute(AZ::Edit::Attributes::SystemComponentTags, AZStd::vector<AZ::Crc32>{AZ_CRC_CE("AssetBuilder")})
  208. ;
  209. if (AZ::EditContext* ec = serialize->GetEditContext())
  210. {
  211. ec->Class<PythonSystemComponent>("PythonSystemComponent", "The Python interpreter")
  212. ->ClassElement(AZ::Edit::ClassElements::EditorData, "")
  213. ->Attribute(AZ::Edit::Attributes::AppearsInAddComponentMenu, AZ_CRC_CE("System"))
  214. ->Attribute(AZ::Edit::Attributes::AutoExpand, true)
  215. ;
  216. }
  217. }
  218. }
  219. void PythonSystemComponent::GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided)
  220. {
  221. provided.push_back(PythonEmbeddedService);
  222. }
  223. void PythonSystemComponent::GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatible)
  224. {
  225. incompatible.push_back(PythonEmbeddedService);
  226. }
  227. void PythonSystemComponent::Activate()
  228. {
  229. AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Register(this);
  230. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusConnect();
  231. }
  232. void PythonSystemComponent::Deactivate()
  233. {
  234. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
  235. AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Unregister(this);
  236. StopPython(true);
  237. }
  238. bool PythonSystemComponent::StartPython([[maybe_unused]] bool silenceWarnings)
  239. {
  240. struct ReleaseInitalizeWaiterScope final
  241. {
  242. using ReleaseFunction = AZStd::function<void(void)>;
  243. ReleaseInitalizeWaiterScope(ReleaseFunction releaseFunction)
  244. {
  245. m_releaseFunction = AZStd::move(releaseFunction);
  246. }
  247. ~ReleaseInitalizeWaiterScope()
  248. {
  249. m_releaseFunction();
  250. }
  251. ReleaseFunction m_releaseFunction;
  252. };
  253. ReleaseInitalizeWaiterScope scope([this]()
  254. {
  255. m_initalizeWaiter.release(m_initalizeWaiterCount);
  256. m_initalizeWaiterCount = 0;
  257. });
  258. if (Py_IsInitialized())
  259. {
  260. AZ_Warning("python", silenceWarnings, "Python is already active!");
  261. return false;
  262. }
  263. PythonPathStack pythonPathStack;
  264. DiscoverPythonPaths(pythonPathStack);
  265. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPreInitialize);
  266. if (StartPythonInterpreter(pythonPathStack))
  267. {
  268. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPostInitialize);
  269. // initialize internal base module and bootstrap scripts
  270. ExecuteByString("import azlmbr", false);
  271. ExecuteBootstrapScripts(pythonPathStack);
  272. return true;
  273. }
  274. return false;
  275. }
  276. bool PythonSystemComponent::StopPython([[maybe_unused]] bool silenceWarnings)
  277. {
  278. if (!Py_IsInitialized())
  279. {
  280. AZ_Warning("python", silenceWarnings, "Python is not active!");
  281. return false;
  282. }
  283. bool result = false;
  284. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPreFinalize);
  285. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
  286. result = StopPythonInterpreter();
  287. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPostFinalize);
  288. return result;
  289. }
  290. void PythonSystemComponent::WaitForInitialization()
  291. {
  292. m_initalizeWaiterCount++;
  293. m_initalizeWaiter.acquire();
  294. }
  295. void PythonSystemComponent::ExecuteWithLock(AZStd::function<void()> executionCallback)
  296. {
  297. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  298. pybind11::gil_scoped_release release;
  299. pybind11::gil_scoped_acquire acquire;
  300. executionCallback();
  301. }
  302. void PythonSystemComponent::DiscoverPythonPaths(PythonPathStack& pythonPathStack)
  303. {
  304. // the order of the Python paths is the order the Python bootstrap scripts will execute
  305. auto settingsRegistry = AZ::SettingsRegistry::Get();
  306. if (!settingsRegistry)
  307. {
  308. return;
  309. }
  310. AZ::IO::FixedMaxPathString projectPath = AZ::Utils::GetProjectPath();
  311. if (projectPath.empty())
  312. {
  313. return;
  314. }
  315. auto resolveScriptPath = [&pythonPathStack](AZStd::string_view path)
  316. {
  317. auto editorScriptsPath = AZ::IO::Path(path) / "Editor" / "Scripts";
  318. if (AZ::IO::SystemFile::Exists(editorScriptsPath.c_str()))
  319. {
  320. pythonPathStack.emplace_back(AZStd::move(editorScriptsPath.LexicallyNormal().Native()));
  321. }
  322. };
  323. // The discovery order will be:
  324. // 1 - engine-root/EngineAsets
  325. // 2 - gems
  326. // 3 - project
  327. // 4 - user(dev)
  328. // 1 - engine
  329. AZ::IO::FixedMaxPath engineRoot;
  330. if (settingsRegistry->Get(engineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder); !engineRoot.empty())
  331. {
  332. resolveScriptPath((engineRoot / "Assets").Native());
  333. }
  334. // 2 - gems
  335. struct GetGemSourcePathsVisitor
  336. : AZ::SettingsRegistryInterface::Visitor
  337. {
  338. GetGemSourcePathsVisitor(AZ::SettingsRegistryInterface& settingsRegistry)
  339. : m_settingsRegistry(settingsRegistry)
  340. {}
  341. void Visit(AZStd::string_view path, AZStd::string_view, AZ::SettingsRegistryInterface::Type,
  342. AZStd::string_view value) override
  343. {
  344. AZStd::string_view jsonSourcePathPointer{ path };
  345. // Remove the array index from the path and check if the JSON path ends with "/SourcePaths"
  346. AZ::StringFunc::TokenizeLast(jsonSourcePathPointer, "/");
  347. if (jsonSourcePathPointer.ends_with("/SourcePaths"))
  348. {
  349. AZ::IO::Path newSourcePath = jsonSourcePathPointer;
  350. // Resolve any file aliases first - Do not use ResolvePath() as that assumes
  351. // any relative path is underneath the @assets@ alias
  352. if (auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); fileIoBase != nullptr)
  353. {
  354. AZ::IO::FixedMaxPath replacedAliasPath;
  355. if (fileIoBase->ReplaceAlias(replacedAliasPath, value))
  356. {
  357. newSourcePath = AZ::IO::PathView(replacedAliasPath);
  358. }
  359. }
  360. // The current assumption is that the gem source path is the relative to the engine root
  361. AZ::IO::Path engineRootPath;
  362. m_settingsRegistry.Get(engineRootPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder);
  363. newSourcePath = (engineRootPath / newSourcePath).LexicallyNormal();
  364. if (auto gemSourcePathIter = AZStd::find(m_gemSourcePaths.begin(), m_gemSourcePaths.end(), newSourcePath);
  365. gemSourcePathIter == m_gemSourcePaths.end())
  366. {
  367. m_gemSourcePaths.emplace_back(AZStd::move(newSourcePath));
  368. }
  369. }
  370. }
  371. AZStd::vector<AZ::IO::Path> m_gemSourcePaths;
  372. private:
  373. AZ::SettingsRegistryInterface& m_settingsRegistry;
  374. };
  375. GetGemSourcePathsVisitor visitor{ *settingsRegistry };
  376. constexpr auto gemListKey = AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::OrganizationRootKey)
  377. + "/Gems";
  378. settingsRegistry->Visit(visitor, gemListKey);
  379. for (const AZ::IO::Path& gemSourcePath : visitor.m_gemSourcePaths)
  380. {
  381. resolveScriptPath(gemSourcePath.Native());
  382. }
  383. // 3 - project
  384. resolveScriptPath(AZStd::string_view{ projectPath });
  385. // 4 - user
  386. AZStd::string assetsType;
  387. AZ::SettingsRegistryMergeUtils::PlatformGet(*settingsRegistry, assetsType,
  388. AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey, AzFramework::AssetSystem::Assets);
  389. if (!assetsType.empty())
  390. {
  391. AZ::IO::FixedMaxPath userCachePath;
  392. if (settingsRegistry->Get(userCachePath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_CacheRootFolder);
  393. !userCachePath.empty())
  394. {
  395. userCachePath /= "user";
  396. resolveScriptPath(userCachePath.Native());
  397. }
  398. }
  399. }
  400. void PythonSystemComponent::ExecuteBootstrapScripts(const PythonPathStack& pythonPathStack)
  401. {
  402. for(const auto& path : pythonPathStack)
  403. {
  404. AZStd::string bootstrapPath;
  405. AzFramework::StringFunc::Path::Join(path.c_str(), "bootstrap.py", bootstrapPath);
  406. if (AZ::IO::SystemFile::Exists(bootstrapPath.c_str()))
  407. {
  408. ExecuteByFilename(bootstrapPath);
  409. }
  410. }
  411. }
  412. bool PythonSystemComponent::StartPythonInterpreter(const PythonPathStack& pythonPathStack)
  413. {
  414. AZStd::unordered_set<AZStd::string> pyPackageSites(pythonPathStack.begin(), pythonPathStack.end());
  415. const char* engineRoot = nullptr;
  416. AzFramework::ApplicationRequests::Bus::BroadcastResult(engineRoot, &AzFramework::ApplicationRequests::GetEngineRoot);
  417. // set PYTHON_HOME
  418. AZStd::string pyBasePath = Platform::GetPythonHomePath(PY_PACKAGE, engineRoot);
  419. if (!AZ::IO::SystemFile::Exists(pyBasePath.c_str()))
  420. {
  421. AZ_Warning("python", false, "Python home path must exist! path:%s", pyBasePath.c_str());
  422. return false;
  423. }
  424. AZStd::wstring pyHomePath;
  425. AZStd::to_wstring(pyHomePath, pyBasePath);
  426. Py_SetPythonHome(pyHomePath.c_str());
  427. // display basic Python information
  428. AZ_TracePrintf("python", "Py_GetVersion=%s \n", Py_GetVersion());
  429. AZ_TracePrintf("python", "Py_GetPath=%ls \n", Py_GetPath());
  430. AZ_TracePrintf("python", "Py_GetExecPrefix=%ls \n", Py_GetExecPrefix());
  431. AZ_TracePrintf("python", "Py_GetProgramFullPath=%ls \n", Py_GetProgramFullPath());
  432. PyImport_AppendInittab("azlmbr_redirect", RedirectOutput::PyInit_RedirectOutput);
  433. try
  434. {
  435. // ignore system location for sites site-packages
  436. Py_IsolatedFlag = 1; // -I - Also sets Py_NoUserSiteDirectory. If removed PyNoUserSiteDirectory should be set.
  437. Py_IgnoreEnvironmentFlag = 1; // -E
  438. const bool initializeSignalHandlers = true;
  439. pybind11::initialize_interpreter(initializeSignalHandlers);
  440. // Add custom site packages after initializing the interpreter above. Calling Py_SetPath before initialization
  441. // alters the behavior of the initializer to not compute default search paths. See https://docs.python.org/3/c-api/init.html#c.Py_SetPath
  442. if (pyPackageSites.size())
  443. {
  444. ExtendSysPath(pyPackageSites);
  445. }
  446. RedirectOutput::Intialize(PyImport_ImportModule("azlmbr_redirect"));
  447. // Acquire GIL before calling Python code
  448. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  449. pybind11::gil_scoped_acquire acquire;
  450. // print Python version using AZ logging
  451. const int verRet = PyRun_SimpleStringFlags("import sys \nprint (sys.version) \n", nullptr);
  452. AZ_Error("python", verRet == 0, "Error trying to fetch the version number in Python!");
  453. return verRet == 0 && !PyErr_Occurred();
  454. }
  455. catch ([[maybe_unused]] const std::exception& e)
  456. {
  457. AZ_Warning("python", false, "Py_Initialize() failed with %s!", e.what());
  458. return false;
  459. }
  460. }
  461. bool PythonSystemComponent::ExtendSysPath(const AZStd::unordered_set<AZStd::string>& extendPaths)
  462. {
  463. AZStd::unordered_set<AZStd::string> oldPathSet;
  464. auto SplitPath = [&oldPathSet](AZStd::string_view pathPart)
  465. {
  466. oldPathSet.emplace(pathPart);
  467. };
  468. AZ::StringFunc::TokenizeVisitor(Py_EncodeLocale(Py_GetPath(), nullptr), SplitPath, DELIM);
  469. bool appended{ false };
  470. AZStd::string pathAppend{ "import sys\n" };
  471. for (const auto& thisStr : extendPaths)
  472. {
  473. if (!oldPathSet.contains(thisStr))
  474. {
  475. pathAppend.append(AZStd::string::format("sys.path.append(r'%s')\n", thisStr.c_str()));
  476. appended = true;
  477. }
  478. }
  479. if (appended)
  480. {
  481. ExecuteByString(pathAppend.c_str(), false);
  482. return true;
  483. }
  484. return false;
  485. }
  486. bool PythonSystemComponent::StopPythonInterpreter()
  487. {
  488. if (Py_IsInitialized())
  489. {
  490. RedirectOutput::Shutdown();
  491. pybind11::finalize_interpreter();
  492. }
  493. else
  494. {
  495. AZ_Warning("python", false, "Did not finalize since Py_IsInitialized() was false.");
  496. }
  497. return !PyErr_Occurred();
  498. }
  499. void PythonSystemComponent::ExecuteByString(AZStd::string_view script, bool printResult)
  500. {
  501. if (!Py_IsInitialized())
  502. {
  503. AZ_Error("python", false, "Can not ExecuteByString() since the embeded Python VM is not ready.");
  504. return;
  505. }
  506. if (!script.empty())
  507. {
  508. // Acquire GIL before calling Python code
  509. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  510. pybind11::gil_scoped_acquire acquire;
  511. // Acquire scope for __main__ for executing our script
  512. pybind11::object scope = pybind11::module::import("__main__").attr("__dict__");
  513. bool shouldPrintValue = false;
  514. if (printResult)
  515. {
  516. // Attempt to compile our code to determine if it's an expression
  517. // i.e. a Python code object with only an rvalue
  518. // If it is, it can be evaled to produce a PyObject
  519. // If it's not, we can't evaluate it into a result and should fall back to exec
  520. shouldPrintValue = true;
  521. using namespace pybind11::literals;
  522. // codeop.compile_command is a thin wrapper around the Python compile builtin
  523. // We attempt to compile using symbol="eval" to see if the string is valid for eval
  524. // This is similar to what the Python REPL does internally
  525. pybind11::object codeop = pybind11::module::import("codeop");
  526. pybind11::object compileCommand = codeop.attr("compile_command");
  527. try
  528. {
  529. compileCommand(script.data(), "symbol"_a="eval");
  530. }
  531. catch (const pybind11::error_already_set&)
  532. {
  533. shouldPrintValue = false;
  534. }
  535. }
  536. try
  537. {
  538. if (shouldPrintValue)
  539. {
  540. // We're an expression, run and print the result
  541. pybind11::object result = pybind11::eval(script.data(), scope);
  542. pybind11::print(result);
  543. }
  544. else
  545. {
  546. // Just exec the code block
  547. pybind11::exec(script.data(), scope);
  548. }
  549. }
  550. catch (pybind11::error_already_set& pythonError)
  551. {
  552. // Release the exception stack and let Python print it to stderr
  553. pythonError.restore();
  554. PyErr_Print();
  555. }
  556. }
  557. }
  558. void PythonSystemComponent::ExecuteByFilename(AZStd::string_view filename)
  559. {
  560. AZStd::vector<AZStd::string_view> args;
  561. ExecuteByFilenameWithArgs(filename, args);
  562. }
  563. void PythonSystemComponent::ExecuteByFilenameAsTest(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  564. {
  565. const Result evalResult = EvaluateFile(filename, args);
  566. if (evalResult == Result::Okay)
  567. {
  568. // all good, the test script will need to exit the application now
  569. return;
  570. }
  571. else
  572. {
  573. // something went wrong with executing the test script
  574. AZ::Debug::Trace::Terminate(0xF);
  575. }
  576. }
  577. void PythonSystemComponent::ExecuteByFilenameWithArgs(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  578. {
  579. EvaluateFile(filename, args);
  580. }
  581. PythonSystemComponent::Result PythonSystemComponent::EvaluateFile(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  582. {
  583. if (!Py_IsInitialized())
  584. {
  585. AZ_Error("python", false, "Can not evaluate file since the embedded Python VM is not ready.");
  586. return Result::Error_IsNotInitialized;
  587. }
  588. if (filename.empty())
  589. {
  590. AZ_Error("python", false, "Invalid empty filename detected.");
  591. return Result::Error_InvalidFilename;
  592. }
  593. // support the alias version of a script such as @devroot@/Editor/Scripts/select_story_anim_objects.py
  594. AZStd::string theFilename(filename);
  595. {
  596. char resolvedPath[AZ_MAX_PATH_LEN] = { 0 };
  597. AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(theFilename.c_str(), resolvedPath, AZ_MAX_PATH_LEN);
  598. theFilename = resolvedPath;
  599. }
  600. if (!AZ::IO::FileIOBase::GetInstance()->Exists(theFilename.c_str()))
  601. {
  602. AZ_Error("python", false, "Missing Python file named (%s)", theFilename.c_str());
  603. return Result::Error_MissingFile;
  604. }
  605. FILE* file = _Py_fopen(theFilename.data(), "rb");
  606. if (!file)
  607. {
  608. AZ_Error("python", false, "Missing Python file named (%s)", theFilename.c_str());
  609. return Result::Error_FileOpenValidation;
  610. }
  611. Result pythonScriptResult = Result::Okay;
  612. try
  613. {
  614. // Acquire GIL before calling Python code
  615. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  616. pybind11::gil_scoped_acquire acquire;
  617. // Create standard "argc" / "argv" command-line parameters to pass in to the Python script via sys.argv.
  618. // argc = number of parameters. This will always be at least 1, since the first parameter is the script name.
  619. // argv = the list of parameters, in wchar format.
  620. // Our expectation is that the args passed into this function does *not* already contain the script name.
  621. int argc = aznumeric_cast<int>(args.size()) + 1;
  622. // Note: This allocates from PyMem to ensure that Python has access to the memory.
  623. wchar_t** argv = static_cast<wchar_t**>(PyMem_Malloc(argc * sizeof(wchar_t*)));
  624. // Python 3.x is expecting wchar* strings for the command-line args.
  625. argv[0] = Py_DecodeLocale(theFilename.c_str(), nullptr);
  626. for (int arg = 0; arg < args.size(); arg++)
  627. {
  628. argv[arg + 1] = Py_DecodeLocale(args[arg].data(), nullptr);
  629. }
  630. // Tell Python the command-line args.
  631. // Note that this has a side effect of adding the script's path to the set of directories checked for "import" commands.
  632. const int updatePath = 1;
  633. PySys_SetArgvEx(argc, argv, updatePath);
  634. PyCompilerFlags flags;
  635. flags.cf_flags = 0;
  636. const int bAutoCloseFile = true;
  637. const int returnCode = PyRun_SimpleFileExFlags(file, theFilename.c_str(), bAutoCloseFile, &flags);
  638. if (returnCode != 0)
  639. {
  640. AZStd::string message = AZStd::string::format("Detected script failure in Python script(%s); return code %d!", theFilename.c_str(), returnCode);
  641. AZ_Warning("python", false, message.c_str());
  642. using namespace AzToolsFramework;
  643. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnExceptionMessage, message.c_str());
  644. pythonScriptResult = Result::Error_PythonException;
  645. }
  646. // Free any memory allocated for the command-line args.
  647. for (int arg = 0; arg < argc; arg++)
  648. {
  649. PyMem_RawFree(argv[arg]);
  650. }
  651. PyMem_Free(argv);
  652. }
  653. catch ([[maybe_unused]] const std::exception& e)
  654. {
  655. AZ_Error("python", false, "Detected an internal exception %s while running script (%s)!", e.what(), theFilename.c_str());
  656. return Result::Error_InternalException;
  657. }
  658. return pythonScriptResult;
  659. }
  660. } // namespace EditorPythonBindings