PythonSystemComponent.cpp 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757
  1. /*
  2. * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
  3. * its licensors.
  4. *
  5. * For complete copyright and license terms please see the LICENSE at the root of this
  6. * distribution (the "License"). All use of this software is governed by the License,
  7. * or, if provided, by the license below or the license accompanying this file. Do not
  8. * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
  9. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. *
  11. */
  12. #include <PythonSystemComponent.h>
  13. #include <EditorPythonBindings/EditorPythonBindingsBus.h>
  14. #include <Source/PythonCommon.h>
  15. #include <pybind11/pybind11.h>
  16. #include <pybind11/embed.h>
  17. #include <pybind11/eval.h>
  18. #include <osdefs.h> // for DELIM
  19. #include <AzCore/Component/EntityId.h>
  20. #include <AzCore/IO/SystemFile.h>
  21. #include <AzCore/Module/DynamicModuleHandle.h>
  22. #include <AzCore/Module/Module.h>
  23. #include <AzCore/Module/ModuleManagerBus.h>
  24. #include <AzCore/PlatformDef.h>
  25. #include <AzCore/Serialization/EditContext.h>
  26. #include <AzCore/Serialization/SerializeContext.h>
  27. #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
  28. #include <AzCore/std/string/conversions.h>
  29. #include <AzCore/StringFunc/StringFunc.h>
  30. #include <AzCore/Utils/Utils.h>
  31. #include <AzFramework/API/ApplicationAPI.h>
  32. #include <AzFramework/Asset/AssetSystemComponent.h>
  33. #include <AzFramework/IO/LocalFileIO.h>
  34. #include <AzFramework/CommandLine/CommandRegistrationBus.h>
  35. #include <AzFramework/StringFunc/StringFunc.h>
  36. #include <AzToolsFramework/API/EditorPythonConsoleBus.h>
  37. namespace Platform
  38. {
  39. // Implemented in each different platform's implentation files, as it differs per platform.
  40. bool InsertPythonBinaryLibraryPaths(AZStd::unordered_set<AZStd::string>& paths, const char* pythonPackage, const char* engineRoot);
  41. AZStd::string GetPythonHomePath(const char* pythonPackage, const char* engineRoot);
  42. }
  43. // this is called the first time a Python script contains "import azlmbr"
  44. PYBIND11_EMBEDDED_MODULE(azlmbr, m)
  45. {
  46. EditorPythonBindings::EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindings::EditorPythonBindingsNotificationBus::Events::OnImportModule, m.ptr());
  47. }
  48. namespace RedirectOutput
  49. {
  50. using RedirectOutputFunc = AZStd::function<void(const char*)>;
  51. struct RedirectOutput
  52. {
  53. PyObject_HEAD
  54. RedirectOutputFunc write;
  55. };
  56. PyObject* RedirectWrite(PyObject* self, PyObject* args)
  57. {
  58. std::size_t written(0);
  59. RedirectOutput* selfimpl = reinterpret_cast<RedirectOutput*>(self);
  60. if (selfimpl->write)
  61. {
  62. char* data;
  63. if (!PyArg_ParseTuple(args, "s", &data))
  64. {
  65. return PyLong_FromSize_t(0);
  66. }
  67. selfimpl->write(data);
  68. written = strlen(data);
  69. }
  70. return PyLong_FromSize_t(written);
  71. }
  72. PyObject* RedirectFlush([[maybe_unused]] PyObject* self, [[maybe_unused]] PyObject* args)
  73. {
  74. // no-op
  75. return Py_BuildValue("");
  76. }
  77. PyMethodDef RedirectMethods[] =
  78. {
  79. {"write", RedirectWrite, METH_VARARGS, "sys.stdout.write"},
  80. {"flush", RedirectFlush, METH_VARARGS, "sys.stdout.flush"},
  81. {"write", RedirectWrite, METH_VARARGS, "sys.stderr.write"},
  82. {"flush", RedirectFlush, METH_VARARGS, "sys.stderr.flush"},
  83. {0, 0, 0, 0} // sentinel
  84. };
  85. PyTypeObject RedirectOutputType =
  86. {
  87. PyVarObject_HEAD_INIT(0, 0)
  88. "azlmbr_redirect.RedirectOutputType", // tp_name
  89. sizeof(RedirectOutput), /* tp_basicsize */
  90. 0, /* tp_itemsize */
  91. 0, /* tp_dealloc */
  92. 0, /* tp_print */
  93. 0, /* tp_getattr */
  94. 0, /* tp_setattr */
  95. 0, /* tp_reserved */
  96. 0, /* tp_repr */
  97. 0, /* tp_as_number */
  98. 0, /* tp_as_sequence */
  99. 0, /* tp_as_mapping */
  100. 0, /* tp_hash */
  101. 0, /* tp_call */
  102. 0, /* tp_str */
  103. 0, /* tp_getattro */
  104. 0, /* tp_setattro */
  105. 0, /* tp_as_buffer */
  106. Py_TPFLAGS_DEFAULT, /* tp_flags */
  107. "azlmbr_redirect objects", /* tp_doc */
  108. 0, /* tp_traverse */
  109. 0, /* tp_clear */
  110. 0, /* tp_richcompare */
  111. 0, /* tp_weaklistoffset */
  112. 0, /* tp_iter */
  113. 0, /* tp_iternext */
  114. RedirectMethods, /* tp_methods */
  115. 0, /* tp_members */
  116. 0, /* tp_getset */
  117. 0, /* tp_base */
  118. 0, /* tp_dict */
  119. 0, /* tp_descr_get */
  120. 0, /* tp_descr_set */
  121. 0, /* tp_dictoffset */
  122. 0, /* tp_init */
  123. 0, /* tp_alloc */
  124. 0 /* tp_new */
  125. };
  126. PyModuleDef RedirectOutputModule = { PyModuleDef_HEAD_INIT, "azlmbr_redirect", 0, -1, 0, };
  127. // Internal state
  128. PyObject* g_redirect_stdout = nullptr;
  129. PyObject* g_redirect_stdout_saved = nullptr;
  130. PyObject* g_redirect_stderr = nullptr;
  131. PyObject* g_redirect_stderr_saved = nullptr;
  132. PyMODINIT_FUNC PyInit_RedirectOutput(void)
  133. {
  134. g_redirect_stdout = nullptr;
  135. g_redirect_stdout_saved = nullptr;
  136. g_redirect_stderr = nullptr;
  137. g_redirect_stderr_saved = nullptr;
  138. RedirectOutputType.tp_new = PyType_GenericNew;
  139. if (PyType_Ready(&RedirectOutputType) < 0)
  140. {
  141. return 0;
  142. }
  143. PyObject* m = PyModule_Create(&RedirectOutputModule);
  144. if (m)
  145. {
  146. Py_INCREF(&RedirectOutputType);
  147. PyModule_AddObject(m, "Redirect", reinterpret_cast<PyObject*>(&RedirectOutputType));
  148. }
  149. return m;
  150. }
  151. void SetRedirection(const char* funcname, PyObject*& saved, PyObject*& current, RedirectOutputFunc func)
  152. {
  153. if (PyType_Ready(&RedirectOutputType) < 0)
  154. {
  155. AZ_Warning("python", false, "RedirectOutputType not ready!");
  156. return;
  157. }
  158. if (!current)
  159. {
  160. saved = PySys_GetObject(funcname); // borrowed
  161. current = RedirectOutputType.tp_new(&RedirectOutputType, 0, 0);
  162. }
  163. RedirectOutput* redirectOutput = reinterpret_cast<RedirectOutput*>(current);
  164. redirectOutput->write = func;
  165. PySys_SetObject(funcname, current);
  166. }
  167. void ResetRedirection(const char* funcname, PyObject*& saved, PyObject*& current)
  168. {
  169. if (current)
  170. {
  171. PySys_SetObject(funcname, saved);
  172. }
  173. Py_XDECREF(current);
  174. current = nullptr;
  175. }
  176. PyObject* s_RedirectModule = nullptr;
  177. void Intialize(PyObject* module)
  178. {
  179. using namespace AzToolsFramework;
  180. s_RedirectModule = module;
  181. SetRedirection("stdout", g_redirect_stdout_saved, g_redirect_stdout, [](const char* msg)
  182. {
  183. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnTraceMessage, msg);
  184. });
  185. SetRedirection("stderr", g_redirect_stderr_saved, g_redirect_stderr, [](const char* msg)
  186. {
  187. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnErrorMessage, msg);
  188. });
  189. PySys_WriteStdout("RedirectOutput installed");
  190. }
  191. void Shutdown()
  192. {
  193. ResetRedirection("stdout", g_redirect_stdout_saved, g_redirect_stdout);
  194. ResetRedirection("stderr", g_redirect_stderr_saved, g_redirect_stderr);
  195. Py_XDECREF(s_RedirectModule);
  196. s_RedirectModule = nullptr;
  197. }
  198. } // namespace RedirectOutput
  199. namespace EditorPythonBindings
  200. {
  201. void PythonSystemComponent::Reflect(AZ::ReflectContext* context)
  202. {
  203. if (AZ::SerializeContext* serialize = azrtti_cast<AZ::SerializeContext*>(context))
  204. {
  205. serialize->Class<PythonSystemComponent, AZ::Component>()
  206. ->Version(1)
  207. ->Attribute(AZ::Edit::Attributes::SystemComponentTags, AZStd::vector<AZ::Crc32>{AZ_CRC_CE("AssetBuilder")})
  208. ;
  209. if (AZ::EditContext* ec = serialize->GetEditContext())
  210. {
  211. ec->Class<PythonSystemComponent>("PythonSystemComponent", "The Python interpreter")
  212. ->ClassElement(AZ::Edit::ClassElements::EditorData, "")
  213. ->Attribute(AZ::Edit::Attributes::AppearsInAddComponentMenu, AZ_CRC_CE("System"))
  214. ->Attribute(AZ::Edit::Attributes::AutoExpand, true)
  215. ;
  216. }
  217. }
  218. }
  219. void PythonSystemComponent::GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided)
  220. {
  221. provided.push_back(PythonEmbeddedService);
  222. }
  223. void PythonSystemComponent::GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatible)
  224. {
  225. incompatible.push_back(PythonEmbeddedService);
  226. }
  227. void PythonSystemComponent::Activate()
  228. {
  229. AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Register(this);
  230. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusConnect();
  231. }
  232. void PythonSystemComponent::Deactivate()
  233. {
  234. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
  235. AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Unregister(this);
  236. StopPython(true);
  237. }
  238. bool PythonSystemComponent::StartPython([[maybe_unused]] bool silenceWarnings)
  239. {
  240. struct ReleaseInitalizeWaiterScope final
  241. {
  242. using ReleaseFunction = AZStd::function<void(void)>;
  243. ReleaseInitalizeWaiterScope(ReleaseFunction releaseFunction)
  244. {
  245. m_releaseFunction = AZStd::move(releaseFunction);
  246. }
  247. ~ReleaseInitalizeWaiterScope()
  248. {
  249. m_releaseFunction();
  250. }
  251. ReleaseFunction m_releaseFunction;
  252. };
  253. ReleaseInitalizeWaiterScope scope([this]()
  254. {
  255. m_initalizeWaiter.release(m_initalizeWaiterCount);
  256. m_initalizeWaiterCount = 0;
  257. });
  258. if (Py_IsInitialized())
  259. {
  260. AZ_Warning("python", silenceWarnings, "Python is already active!");
  261. return false;
  262. }
  263. PythonPathStack pythonPathStack;
  264. DiscoverPythonPaths(pythonPathStack);
  265. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPreInitialize);
  266. if (StartPythonInterpreter(pythonPathStack))
  267. {
  268. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPostInitialize);
  269. // initialize internal base module and bootstrap scripts
  270. ExecuteByString("import azlmbr", false);
  271. ExecuteBootstrapScripts(pythonPathStack);
  272. return true;
  273. }
  274. return false;
  275. }
  276. bool PythonSystemComponent::StopPython([[maybe_unused]] bool silenceWarnings)
  277. {
  278. if (!Py_IsInitialized())
  279. {
  280. AZ_Warning("python", silenceWarnings, "Python is not active!");
  281. return false;
  282. }
  283. bool result = false;
  284. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPreFinalize);
  285. AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
  286. result = StopPythonInterpreter();
  287. EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPostFinalize);
  288. return result;
  289. }
  290. void PythonSystemComponent::WaitForInitialization()
  291. {
  292. m_initalizeWaiterCount++;
  293. m_initalizeWaiter.acquire();
  294. }
  295. void PythonSystemComponent::ExecuteWithLock(AZStd::function<void()> executionCallback)
  296. {
  297. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  298. pybind11::gil_scoped_release release;
  299. pybind11::gil_scoped_acquire acquire;
  300. executionCallback();
  301. }
  302. void PythonSystemComponent::DiscoverPythonPaths(PythonPathStack& pythonPathStack)
  303. {
  304. // the order of the Python paths is the order the Python bootstrap scripts will execute
  305. auto settingsRegistry = AZ::SettingsRegistry::Get();
  306. if (!settingsRegistry)
  307. {
  308. return;
  309. }
  310. AZ::IO::FixedMaxPathString projectPath = AZ::Utils::GetProjectPath();
  311. if (projectPath.empty())
  312. {
  313. return;
  314. }
  315. auto resolveScriptPath = [&pythonPathStack](AZStd::string_view path)
  316. {
  317. AZ::IO::Path editorScriptsPath(path);
  318. editorScriptsPath /= "Editor/Scripts";
  319. if (AZ::IO::SystemFile::Exists(editorScriptsPath.c_str()))
  320. {
  321. pythonPathStack.emplace_back(AZStd::move(editorScriptsPath.LexicallyNormal().Native()));
  322. }
  323. };
  324. // The discovery order will be:
  325. // 1 - engine
  326. // 2 - gems
  327. // 3 - project
  328. // 4 - user(dev)
  329. // 1 - engine
  330. AZ::IO::FixedMaxPath engineRoot;
  331. if (settingsRegistry->Get(engineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder); !engineRoot.empty())
  332. {
  333. resolveScriptPath(engineRoot.Native());
  334. }
  335. // 2 - gems
  336. struct GetGemSourcePathsVisitor
  337. : AZ::SettingsRegistryInterface::Visitor
  338. {
  339. GetGemSourcePathsVisitor(AZ::SettingsRegistryInterface& settingsRegistry)
  340. : m_settingsRegistry(settingsRegistry)
  341. {}
  342. void Visit(AZStd::string_view path, AZStd::string_view, AZ::SettingsRegistryInterface::Type,
  343. AZStd::string_view value) override
  344. {
  345. AZStd::string_view jsonSourcePathPointer{ path };
  346. // Remove the array index from the path and check if the JSON path ends with "/SourcePaths"
  347. AZ::StringFunc::TokenizeLast(jsonSourcePathPointer, "/");
  348. if (jsonSourcePathPointer.ends_with("/SourcePaths"))
  349. {
  350. AZ::IO::Path newSourcePath = jsonSourcePathPointer;
  351. // Resolve any file aliases first - Do not use ResolvePath() as that assumes
  352. // any relative path is underneath the @assets@ alias
  353. if (auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); fileIoBase != nullptr)
  354. {
  355. AZ::IO::FixedMaxPath replacedAliasPath;
  356. if (fileIoBase->ReplaceAlias(replacedAliasPath, value))
  357. {
  358. newSourcePath = AZ::IO::PathView(replacedAliasPath);
  359. }
  360. }
  361. // The current assumption is that the gem source path is the relative to the engine root
  362. AZ::IO::Path engineRootPath;
  363. m_settingsRegistry.Get(engineRootPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder);
  364. newSourcePath = (engineRootPath / newSourcePath).LexicallyNormal();
  365. if (auto gemSourcePathIter = AZStd::find(m_gemSourcePaths.begin(), m_gemSourcePaths.end(), newSourcePath);
  366. gemSourcePathIter == m_gemSourcePaths.end())
  367. {
  368. m_gemSourcePaths.emplace_back(AZStd::move(newSourcePath));
  369. }
  370. }
  371. }
  372. AZStd::vector<AZ::IO::Path> m_gemSourcePaths;
  373. private:
  374. AZ::SettingsRegistryInterface& m_settingsRegistry;
  375. };
  376. GetGemSourcePathsVisitor visitor{ *settingsRegistry };
  377. constexpr auto gemListKey = AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::OrganizationRootKey)
  378. + "/Gems";
  379. settingsRegistry->Visit(visitor, gemListKey);
  380. for (const AZ::IO::Path& gemSourcePath : visitor.m_gemSourcePaths)
  381. {
  382. resolveScriptPath(gemSourcePath.Native());
  383. }
  384. // 3 - project
  385. resolveScriptPath(AZStd::string_view{ projectPath });
  386. // 4 - user
  387. AZStd::string assetsType;
  388. AZ::SettingsRegistryMergeUtils::PlatformGet(*settingsRegistry, assetsType,
  389. AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey, AzFramework::AssetSystem::Assets);
  390. if (!assetsType.empty())
  391. {
  392. AZ::IO::FixedMaxPath userCachePath;
  393. if (settingsRegistry->Get(userCachePath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_CacheRootFolder);
  394. !userCachePath.empty())
  395. {
  396. userCachePath /= "user";
  397. resolveScriptPath(userCachePath.Native());
  398. }
  399. }
  400. }
  401. void PythonSystemComponent::ExecuteBootstrapScripts(const PythonPathStack& pythonPathStack)
  402. {
  403. for(const auto& path : pythonPathStack)
  404. {
  405. AZStd::string bootstrapPath;
  406. AzFramework::StringFunc::Path::Join(path.c_str(), "bootstrap.py", bootstrapPath);
  407. if (AZ::IO::SystemFile::Exists(bootstrapPath.c_str()))
  408. {
  409. ExecuteByFilename(bootstrapPath);
  410. }
  411. }
  412. }
  413. bool PythonSystemComponent::StartPythonInterpreter(const PythonPathStack& pythonPathStack)
  414. {
  415. AZStd::unordered_set<AZStd::string> pyPackageSites(pythonPathStack.begin(), pythonPathStack.end());
  416. const char* engineRoot = nullptr;
  417. AzFramework::ApplicationRequests::Bus::BroadcastResult(engineRoot, &AzFramework::ApplicationRequests::GetEngineRoot);
  418. // set PYTHON_HOME
  419. AZStd::string pyBasePath = Platform::GetPythonHomePath(PY_PACKAGE, engineRoot);
  420. if (!AZ::IO::SystemFile::Exists(pyBasePath.c_str()))
  421. {
  422. AZ_Warning("python", false, "Python home path must exist! path:%s", pyBasePath.c_str());
  423. return false;
  424. }
  425. AZStd::wstring pyHomePath;
  426. AZStd::to_wstring(pyHomePath, pyBasePath);
  427. Py_SetPythonHome(pyHomePath.c_str());
  428. // display basic Python information
  429. AZ_TracePrintf("python", "Py_GetVersion=%s \n", Py_GetVersion());
  430. AZ_TracePrintf("python", "Py_GetPath=%ls \n", Py_GetPath());
  431. AZ_TracePrintf("python", "Py_GetExecPrefix=%ls \n", Py_GetExecPrefix());
  432. AZ_TracePrintf("python", "Py_GetProgramFullPath=%ls \n", Py_GetProgramFullPath());
  433. PyImport_AppendInittab("azlmbr_redirect", RedirectOutput::PyInit_RedirectOutput);
  434. try
  435. {
  436. // ignore system location for sites site-packages
  437. Py_IsolatedFlag = 1; // -I - Also sets Py_NoUserSiteDirectory. If removed PyNoUserSiteDirectory should be set.
  438. Py_IgnoreEnvironmentFlag = 1; // -E
  439. const bool initializeSignalHandlers = true;
  440. pybind11::initialize_interpreter(initializeSignalHandlers);
  441. // Add custom site packages after initializing the interpreter above. Calling Py_SetPath before initialization
  442. // alters the behavior of the initializer to not compute default search paths. See https://docs.python.org/3/c-api/init.html#c.Py_SetPath
  443. if (pyPackageSites.size())
  444. {
  445. ExtendSysPath(pyPackageSites);
  446. }
  447. RedirectOutput::Intialize(PyImport_ImportModule("azlmbr_redirect"));
  448. // Acquire GIL before calling Python code
  449. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  450. pybind11::gil_scoped_acquire acquire;
  451. // print Python version using AZ logging
  452. const int verRet = PyRun_SimpleStringFlags("import sys \nprint (sys.version) \n", nullptr);
  453. AZ_Error("python", verRet == 0, "Error trying to fetch the version number in Python!");
  454. return verRet == 0 && !PyErr_Occurred();
  455. }
  456. catch ([[maybe_unused]] const std::exception& e)
  457. {
  458. AZ_Warning("python", false, "Py_Initialize() failed with %s!", e.what());
  459. return false;
  460. }
  461. }
  462. bool PythonSystemComponent::ExtendSysPath(const AZStd::unordered_set<AZStd::string>& extendPaths)
  463. {
  464. AZStd::unordered_set<AZStd::string> oldPathSet;
  465. auto SplitPath = [&oldPathSet](AZStd::string_view pathPart)
  466. {
  467. oldPathSet.emplace(pathPart);
  468. };
  469. AZ::StringFunc::TokenizeVisitor(Py_EncodeLocale(Py_GetPath(), nullptr), SplitPath, DELIM);
  470. bool appended{ false };
  471. AZStd::string pathAppend{ "import sys\n" };
  472. for (const auto& thisStr : extendPaths)
  473. {
  474. if (!oldPathSet.contains(thisStr))
  475. {
  476. pathAppend.append(AZStd::string::format("sys.path.append('%s')\n", thisStr.c_str()));
  477. appended = true;
  478. }
  479. }
  480. if (appended)
  481. {
  482. ExecuteByString(pathAppend.c_str(), false);
  483. return true;
  484. }
  485. return false;
  486. }
  487. bool PythonSystemComponent::StopPythonInterpreter()
  488. {
  489. if (Py_IsInitialized())
  490. {
  491. RedirectOutput::Shutdown();
  492. pybind11::finalize_interpreter();
  493. }
  494. else
  495. {
  496. AZ_Warning("python", false, "Did not finalize since Py_IsInitialized() was false.");
  497. }
  498. return !PyErr_Occurred();
  499. }
  500. void PythonSystemComponent::ExecuteByString(AZStd::string_view script, bool printResult)
  501. {
  502. if (!Py_IsInitialized())
  503. {
  504. AZ_Error("python", false, "Can not ExecuteByString() since the embeded Python VM is not ready.");
  505. return;
  506. }
  507. if (!script.empty())
  508. {
  509. // Acquire GIL before calling Python code
  510. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  511. pybind11::gil_scoped_acquire acquire;
  512. // Acquire scope for __main__ for executing our script
  513. pybind11::object scope = pybind11::module::import("__main__").attr("__dict__");
  514. bool shouldPrintValue = false;
  515. if (printResult)
  516. {
  517. // Attempt to compile our code to determine if it's an expression
  518. // i.e. a Python code object with only an rvalue
  519. // If it is, it can be evaled to produce a PyObject
  520. // If it's not, we can't evaluate it into a result and should fall back to exec
  521. shouldPrintValue = true;
  522. using namespace pybind11::literals;
  523. // codeop.compile_command is a thin wrapper around the Python compile builtin
  524. // We attempt to compile using symbol="eval" to see if the string is valid for eval
  525. // This is similar to what the Python REPL does internally
  526. pybind11::object codeop = pybind11::module::import("codeop");
  527. pybind11::object compileCommand = codeop.attr("compile_command");
  528. try
  529. {
  530. compileCommand(script.data(), "symbol"_a="eval");
  531. }
  532. catch (const pybind11::error_already_set&)
  533. {
  534. shouldPrintValue = false;
  535. }
  536. }
  537. try
  538. {
  539. if (shouldPrintValue)
  540. {
  541. // We're an expression, run and print the result
  542. pybind11::object result = pybind11::eval(script.data(), scope);
  543. pybind11::print(result);
  544. }
  545. else
  546. {
  547. // Just exec the code block
  548. pybind11::exec(script.data(), scope);
  549. }
  550. }
  551. catch (pybind11::error_already_set& pythonError)
  552. {
  553. // Release the exception stack and let Python print it to stderr
  554. pythonError.restore();
  555. PyErr_Print();
  556. }
  557. }
  558. }
  559. void PythonSystemComponent::ExecuteByFilename(AZStd::string_view filename)
  560. {
  561. AZStd::vector<AZStd::string_view> args;
  562. ExecuteByFilenameWithArgs(filename, args);
  563. }
  564. void PythonSystemComponent::ExecuteByFilenameAsTest(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  565. {
  566. const Result evalResult = EvaluateFile(filename, args);
  567. if (evalResult == Result::Okay)
  568. {
  569. // all good, the test script will need to exit the application now
  570. return;
  571. }
  572. else
  573. {
  574. // something when wrong with executing the test script
  575. AZ::Debug::Trace::Terminate(1);
  576. }
  577. }
  578. void PythonSystemComponent::ExecuteByFilenameWithArgs(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  579. {
  580. EvaluateFile(filename, args);
  581. }
  582. PythonSystemComponent::Result PythonSystemComponent::EvaluateFile(AZStd::string_view filename, const AZStd::vector<AZStd::string_view>& args)
  583. {
  584. if (!Py_IsInitialized())
  585. {
  586. AZ_Error("python", false, "Can not evaluate file since the embedded Python VM is not ready.");
  587. return Result::Error_IsNotInitialized;
  588. }
  589. if (filename.empty())
  590. {
  591. AZ_Error("python", false, "Invalid empty filename detected.");
  592. return Result::Error_InvalidFilename;
  593. }
  594. // support the alias version of a script such as @devroot@/Editor/Scripts/select_story_anim_objects.py
  595. AZStd::string theFilename(filename);
  596. {
  597. char resolvedPath[AZ_MAX_PATH_LEN] = { 0 };
  598. AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(theFilename.c_str(), resolvedPath, AZ_MAX_PATH_LEN);
  599. theFilename = resolvedPath;
  600. }
  601. if (!AZ::IO::FileIOBase::GetInstance()->Exists(theFilename.c_str()))
  602. {
  603. AZ_Error("python", false, "Missing Python file named (%s)", theFilename.c_str());
  604. return Result::Error_MissingFile;
  605. }
  606. FILE* file = _Py_fopen(theFilename.data(), "rb");
  607. if (!file)
  608. {
  609. AZ_Error("python", false, "Missing Python file named (%s)", theFilename.c_str());
  610. return Result::Error_FileOpenValidation;
  611. }
  612. Result pythonScriptResult = Result::Okay;
  613. try
  614. {
  615. // Acquire GIL before calling Python code
  616. AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
  617. pybind11::gil_scoped_acquire acquire;
  618. // Create standard "argc" / "argv" command-line parameters to pass in to the Python script via sys.argv.
  619. // argc = number of parameters. This will always be at least 1, since the first parameter is the script name.
  620. // argv = the list of parameters, in wchar format.
  621. // Our expectation is that the args passed into this function does *not* already contain the script name.
  622. int argc = aznumeric_cast<int>(args.size()) + 1;
  623. // Note: This allocates from PyMem to ensure that Python has access to the memory.
  624. wchar_t** argv = static_cast<wchar_t**>(PyMem_Malloc(argc * sizeof(wchar_t*)));
  625. // Python 3.x is expecting wchar* strings for the command-line args.
  626. argv[0] = Py_DecodeLocale(theFilename.c_str(), nullptr);
  627. for (int arg = 0; arg < args.size(); arg++)
  628. {
  629. argv[arg + 1] = Py_DecodeLocale(args[arg].data(), nullptr);
  630. }
  631. // Tell Python the command-line args.
  632. // Note that this has a side effect of adding the script's path to the set of directories checked for "import" commands.
  633. const int updatePath = 1;
  634. PySys_SetArgvEx(argc, argv, updatePath);
  635. PyCompilerFlags flags;
  636. flags.cf_flags = 0;
  637. const int bAutoCloseFile = true;
  638. const int returnCode = PyRun_SimpleFileExFlags(file, theFilename.c_str(), bAutoCloseFile, &flags);
  639. if (returnCode != 0)
  640. {
  641. AZStd::string message = AZStd::string::format("Detected script failure in Python script(%s); return code %d!", theFilename.c_str(), returnCode);
  642. AZ_Warning("python", false, message.c_str());
  643. using namespace AzToolsFramework;
  644. EditorPythonConsoleNotificationBus::Broadcast(&EditorPythonConsoleNotificationBus::Events::OnExceptionMessage, message.c_str());
  645. pythonScriptResult = Result::Error_PythonException;
  646. }
  647. // Free any memory allocated for the command-line args.
  648. for (int arg = 0; arg < argc; arg++)
  649. {
  650. PyMem_RawFree(argv[arg]);
  651. }
  652. PyMem_Free(argv);
  653. }
  654. catch ([[maybe_unused]] const std::exception& e)
  655. {
  656. AZ_Error("python", false, "Detected an internal exception %s while running script (%s)!", e.what(), theFilename.c_str());
  657. return Result::Error_InternalException;
  658. }
  659. return pythonScriptResult;
  660. }
  661. } // namespace EditorPythonBindings