Speed up GetProjects() and set local "engine_path" in user/project.json (#14657)

* Speed up GetProjects(), merge user/project.json

Signed-off-by: Alex Peterson <[email protected]>

* Remove "engine" field so scan up engine is default

When the "engine" name is set, any engine named "o3de" is a valid engine which means if you have multiple engines installed with that name the first one found is selected.  With AutomatedTesting, we always want to use the engine it comes with regardless of the engine name or version

Signed-off-by: Alex Peterson <[email protected]>

* Fix project properties engine display

Show when a project uses an unregistered engine, or no engine is specified

Signed-off-by: Alex Peterson <[email protected]>

* move get engines json data logic to manifest

Signed-off-by: Alex Peterson <[email protected]>

* disable confusing yellow text under projects

Signed-off-by: Alex Peterson <[email protected]>

* Show project version

Signed-off-by: Alex Peterson <[email protected]>

* Add project version field and display

Signed-off-by: Alex Peterson <[email protected]>

* remove unused imports

Signed-off-by: Alex Peterson <[email protected]>

* remove unnecessary OrderedDict()

Signed-off-by: Alex Peterson <[email protected]>

---------

Signed-off-by: Alex Peterson <[email protected]>

AutomatedTesting/project.json

@@ -5,7 +5,6 @@
     "executable_name": "AutomatedTestingLauncher",
     "modules": [],
     "project_id": "{D816AFAE-4BB7-4FEF-88F4-E2B786DCF29D}",
-    "engine": "o3de",
     "display_name": "AutomatedTesting",
     "icon_path": "preview.png",
     "external_subdirectories": [

Code/Tools/ProjectManager/Resources/ProjectManager.qss

@@ -710,7 +710,7 @@ QTabBar::tab:focus {
 }
 
 #projectButton QLabel#otherEngineLabel {
-    color: #dd0;
+    color: #ddd;
 }
 
 #labelButton QPushButton {

Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp

@@ -403,6 +403,14 @@ namespace O3DE::ProjectManager
     void ProjectButton::SetEngine(const EngineInfo& engine)
     {
         m_engineInfo = engine;
+
+        if (m_engineInfo.m_name.isEmpty() && !m_projectInfo.m_engineName.isEmpty())
+        {
+            // this project wants to use an engine that wasn't found, display the qualifier
+            m_engineInfo.m_name = m_projectInfo.m_engineName;
+            m_engineInfo.m_version = "";
+        }
+
         m_engineNameLabel->SetText(m_engineInfo.m_name + " " + m_engineInfo.m_version);
         m_engineNameLabel->update();
         m_engineNameLabel->setObjectName(m_engineInfo.m_thisEngine ? "thisEngineLabel" : "otherEngineLabel");
@@ -413,7 +421,14 @@ namespace O3DE::ProjectManager
     void ProjectButton::SetProject(const ProjectInfo& project)
     {
         m_projectInfo = project;
-        m_projectNameLabel->SetText(m_projectInfo.GetProjectDisplayName());
+        if (!m_projectInfo.m_version.isEmpty())
+        {
+            m_projectNameLabel->SetText(m_projectInfo.GetProjectDisplayName() + " " + m_projectInfo.m_version);
+        }
+        else
+        {
+            m_projectNameLabel->SetText(m_projectInfo.GetProjectDisplayName());
+        }
         m_projectNameLabel->update();
         m_projectNameLabel->setToolTip(m_projectInfo.m_path);
         m_projectNameLabel->refreshStyle(); // important for styles to work correctly

Code/Tools/ProjectManager/Source/ProjectInfo.cpp

@@ -81,6 +81,10 @@ namespace O3DE::ProjectManager
         {
             return false;
         }
+        if (m_version != rhs.m_version)
+        {
+            return false;
+        }
 
         return true;
     }

Code/Tools/ProjectManager/Source/ProjectInfo.h

@@ -45,6 +45,7 @@ namespace O3DE::ProjectManager
         // From project.json
         QString m_projectName;
         QString m_displayName;
+        QString m_version;
         QString m_engineName;
         QString m_enginePath;
         QString m_id;

Code/Tools/ProjectManager/Source/ProjectSettingsScreen.cpp

@@ -49,6 +49,9 @@ namespace O3DE::ProjectManager
         connect(m_projectName->lineEdit(), &QLineEdit::textChanged, this, &ProjectSettingsScreen::OnProjectNameUpdated);
         m_verticalLayout->addWidget(m_projectName);
 
+        m_projectVersion = new FormLineEditWidget(tr("Project version"), "1.0.0", this);
+        m_verticalLayout->addWidget(m_projectVersion);
+
         m_projectPath = new FormFolderBrowseEditWidget(tr("Project Location"), "", this);
         connect(m_projectPath->lineEdit(), &QLineEdit::textChanged, this, &ProjectSettingsScreen::OnProjectPathUpdated);
         m_verticalLayout->addWidget(m_projectPath);
@@ -84,6 +87,7 @@ namespace O3DE::ProjectManager
     {
         ProjectInfo projectInfo;
         projectInfo.m_projectName = m_projectName->lineEdit()->text();
+        projectInfo.m_version = m_projectVersion->lineEdit()->text();
         // currently we don't have separate fields for changing the project name and display name 
         projectInfo.m_displayName = projectInfo.m_projectName;
         projectInfo.m_path = m_projectPath->lineEdit()->text();

Code/Tools/ProjectManager/Source/ProjectSettingsScreen.h

@@ -47,6 +47,7 @@ namespace O3DE::ProjectManager
         QHBoxLayout* m_horizontalLayout;
         QVBoxLayout* m_verticalLayout;
         FormLineEditWidget* m_projectName;
+        FormLineEditWidget* m_projectVersion;
         FormBrowseEditWidget* m_projectPath;
     };
 

Code/Tools/ProjectManager/Source/ProjectsScreen.cpp

@@ -223,7 +223,7 @@ namespace O3DE::ProjectManager
         auto remoteProjectsResult = PythonBindingsInterface::Get()->GetProjectsForAllRepos();
         if (remoteProjectsResult.IsSuccess() && !remoteProjectsResult.GetValue().isEmpty())
         {
-            const QVector<ProjectInfo>& remoteProjects = remoteProjectsResult.GetValue();
+            const QVector<ProjectInfo>& remoteProjects{ remoteProjectsResult.TakeValue() };
             for (const ProjectInfo& remoteProject : remoteProjects)
             {
                 auto foundProject = AZStd::ranges::find_if(
@@ -309,6 +309,11 @@ namespace O3DE::ProjectManager
                 }
             });
 
+
+            // It's more efficient to update the project engine by loading engine infos once
+            // instead of loading them all each time we want to know what project an engine uses
+            auto engineInfoResult = PythonBindingsInterface::Get()->GetAllEngineInfos();
+
             // Add all project buttons, restoring buttons to default state
             for (const ProjectInfo& project : projects)
             {
@@ -317,9 +322,18 @@ namespace O3DE::ProjectManager
                 auto projectButtonIter = m_projectButtons.find(projectPath);
 
                 EngineInfo engine{};
-                if (auto result = PythonBindingsInterface::Get()->GetProjectEngine(project.m_path); result)
+                if (engineInfoResult && !project.m_enginePath.isEmpty())
                 {
-                    engine = result.GetValue<EngineInfo>();
+                    AZ::IO::FixedMaxPath projectEnginePath{ project.m_enginePath.toUtf8().constData() };
+                    for (const EngineInfo& engineInfo : engineInfoResult.GetValue())
+                    {
+                        AZ::IO::FixedMaxPath enginePath{ engineInfo.m_path.toUtf8().constData() };
+                        if (enginePath == projectEnginePath)
+                        {
+                            engine = engineInfo;
+                            break;
+                        }
+                    }
                 }
 
                 if (projectButtonIter == m_projectButtons.end())

Code/Tools/ProjectManager/Source/PythonBindings.cpp

@@ -335,6 +335,7 @@ namespace O3DE::ProjectManager
             m_enableGemProject = pybind11::module::import("o3de.enable_gem");
             m_disableGemProject = pybind11::module::import("o3de.disable_gem");
             m_editProjectProperties = pybind11::module::import("o3de.project_properties");
+            m_projectManagerInterface = pybind11::module::import("o3de.project_manager_interface");
             m_download = pybind11::module::import("o3de.download");
             m_repo = pybind11::module::import("o3de.repo");
             m_pathlib = pybind11::module::import("pathlib");
@@ -842,6 +843,7 @@ namespace O3DE::ProjectManager
                 "project_path"_a = projectPath,
                 "project_name"_a = QString_To_Py_String(projectInfo.m_projectName),
                 "template_path"_a = QString_To_Py_Path(projectTemplatePath),
+                "version"_a = QString_To_Py_String(projectInfo.m_version),
                 "no_register"_a = !registerProject
             );
             if (createProjectResult.cast<int>() == 0)
@@ -1061,6 +1063,60 @@ namespace O3DE::ProjectManager
         return gemInfo;
     }
 
+    ProjectInfo PythonBindings::ProjectInfoFromDict(pybind11::handle projectData, const QString& path)
+    {
+        ProjectInfo projectInfo;
+        projectInfo.m_needsBuild = false;
+
+        if (!path.isEmpty())
+        {
+            projectInfo.m_path = path;
+        }
+        else
+        {
+            projectInfo.m_path = Py_To_String_Optional(projectData, "path", projectInfo.m_path);
+        }
+
+        projectInfo.m_projectName = Py_To_String(projectData["project_name"]);
+        projectInfo.m_displayName = Py_To_String_Optional(projectData, "display_name", projectInfo.m_projectName);
+        projectInfo.m_version = Py_To_String_Optional(projectData, "version", projectInfo.m_version);
+        projectInfo.m_id = Py_To_String_Optional(projectData, "project_id", projectInfo.m_id);
+        projectInfo.m_origin = Py_To_String_Optional(projectData, "origin", projectInfo.m_origin);
+        projectInfo.m_summary = Py_To_String_Optional(projectData, "summary", projectInfo.m_summary);
+        projectInfo.m_requirements = Py_To_String_Optional(projectData, "requirements", projectInfo.m_requirements);
+        projectInfo.m_license = Py_To_String_Optional(projectData, "license", projectInfo.m_license);
+        projectInfo.m_iconPath = Py_To_String_Optional(projectData, "icon", ProjectPreviewImagePath);
+        projectInfo.m_engineName = Py_To_String_Optional(projectData, "engine", projectInfo.m_engineName);
+        if (projectData.contains("user_tags"))
+        {
+            for (auto tag : projectData["user_tags"])
+            {
+                projectInfo.m_userTags.append(Py_To_String(tag));
+            }
+        }
+
+        if (projectData.contains("engine_path"))
+        {
+            // Python looked for an engine path so we don't need to, but be careful
+            // not to add 'None' in case no path was found
+            if (!pybind11::isinstance<pybind11::none>(projectData["engine_path"]))
+            {
+                projectInfo.m_enginePath = Py_To_String(projectData["engine_path"]);
+            }
+        }
+        else
+        {
+            auto enginePathResult = m_manifest.attr("get_project_engine_path")(QString_To_Py_Path(projectInfo.m_path));
+            if (!pybind11::isinstance<pybind11::none>(enginePathResult))
+            {
+                // request a posix path so it looks like what is in o3de_manifest.json
+                projectInfo.m_enginePath = Py_To_String(enginePathResult.attr("as_posix")());
+            }
+        }
+
+        return projectInfo;
+    }
+
     ProjectInfo PythonBindings::ProjectInfoFromPath(pybind11::handle path)
     {
         ProjectInfo projectInfo;
@@ -1072,29 +1128,7 @@ namespace O3DE::ProjectManager
         {
             try
             {
-                projectInfo.m_projectName = Py_To_String(projectData["project_name"]);
-                projectInfo.m_displayName = Py_To_String_Optional(projectData, "display_name", projectInfo.m_projectName);
-                projectInfo.m_id = Py_To_String_Optional(projectData, "project_id", projectInfo.m_id);
-                projectInfo.m_origin = Py_To_String_Optional(projectData, "origin", projectInfo.m_origin);
-                projectInfo.m_summary = Py_To_String_Optional(projectData, "summary", projectInfo.m_summary);
-                projectInfo.m_requirements = Py_To_String_Optional(projectData, "requirements", projectInfo.m_requirements);
-                projectInfo.m_license = Py_To_String_Optional(projectData, "license", projectInfo.m_license);
-                projectInfo.m_iconPath = Py_To_String_Optional(projectData, "icon", ProjectPreviewImagePath);
-                projectInfo.m_engineName = Py_To_String_Optional(projectData, "engine", projectInfo.m_engineName);
-                if (projectData.contains("user_tags"))
-                {
-                    for (auto tag : projectData["user_tags"])
-                    {
-                        projectInfo.m_userTags.append(Py_To_String(tag));
-                    }
-                }
-
-                auto enginePathResult = m_manifest.attr("get_project_engine_path")(path);
-                if (!pybind11::isinstance<pybind11::none>(enginePathResult))
-                {
-                    // request a posix path so it looks like what is in o3de_manifest.json
-                    projectInfo.m_enginePath = Py_To_String(enginePathResult.attr("as_posix")());
-                }
+                projectInfo = ProjectInfoFromDict(projectData, projectInfo.m_path);
             }
             catch ([[maybe_unused]] const std::exception& e)
             {
@@ -1110,17 +1144,14 @@ namespace O3DE::ProjectManager
         QVector<ProjectInfo> projects;
 
         bool result = ExecuteWithLock([&] {
-            // external projects
-            for (auto path : m_manifest.attr("get_manifest_projects")())
+            for (auto projectData : m_projectManagerInterface.attr("get_all_project_infos")())
             {
-                projects.push_back(ProjectInfoFromPath(path));
+                if (pybind11::isinstance<pybind11::dict>(projectData))
+                {
+                    projects.push_back(ProjectInfoFromDict(projectData));
+                }
             }
 
-            // projects from the engine
-            for (auto path : m_manifest.attr("get_engine_projects")())
-            {
-                projects.push_back(ProjectInfoFromPath(path));
-            }
         });
 
         if (!result)
@@ -1252,9 +1283,18 @@ namespace O3DE::ProjectManager
                     "new_summary"_a = QString_To_Py_String(projectInfo.m_summary),
                     "new_icon"_a = QString_To_Py_String(projectInfo.m_iconPath),
                     "replace_tags"_a = pybind11::list(pybind11::cast(newTags)),
-                    "new_engine_name"_a = QString_To_Py_String(projectInfo.m_engineName)
+                    "new_engine_name"_a = QString_To_Py_String(projectInfo.m_engineName),
+                    "new_version"_a = QString_To_Py_String(projectInfo.m_version)
                     );
                 updateProjectSucceeded = (editResult.cast<int>() == 0);
+
+                // use the specific path locally until we have a UX for specifying the engine version 
+                auto userEditResult = m_editProjectProperties.attr("edit_project_props")(
+                    "proj_path"_a = QString_To_Py_Path(projectInfo.m_path),
+                    "new_engine_path"_a = QString_To_Py_Path(projectInfo.m_enginePath),
+                    "user"_a = true
+                    );
+                updateProjectSucceeded &= (userEditResult.cast<int>() == 0);
             });
 
         if (!result.IsSuccess())

Code/Tools/ProjectManager/Source/PythonBindings.h

@@ -104,6 +104,7 @@ namespace O3DE::ProjectManager
         GemInfo GemInfoFromPath(pybind11::handle path, pybind11::handle pyProjectPath);
         GemRepoInfo GetGemRepoInfo(pybind11::handle repoUri);
         ProjectInfo ProjectInfoFromPath(pybind11::handle path);
+        ProjectInfo ProjectInfoFromDict(pybind11::handle projectData, const QString& path = {});
         ProjectTemplateInfo ProjectTemplateInfoFromPath(pybind11::handle path) const;
         TemplateInfo TemplateInfoFromPath(pybind11::handle path) const;
         AZ::Outcome<void, AZStd::string> GemRegistration(const QString& gemPath, const QString& projectPath, bool remove = false);
@@ -124,6 +125,7 @@ namespace O3DE::ProjectManager
         pybind11::handle m_enableGemProject;
         pybind11::handle m_disableGemProject;
         pybind11::handle m_editProjectProperties;
+        pybind11::handle m_projectManagerInterface;
         pybind11::handle m_download;
         pybind11::handle m_repo;
         pybind11::handle m_pathlib;

Code/Tools/ProjectManager/Source/UpdateProjectSettingsScreen.cpp

@@ -111,6 +111,7 @@ namespace O3DE::ProjectManager
     ProjectInfo UpdateProjectSettingsScreen::GetProjectInfo()
     {
         m_projectInfo.m_displayName = m_projectName->lineEdit()->text();
+        m_projectInfo.m_version = m_projectVersion->lineEdit()->text();
         m_projectInfo.m_path = m_projectPath->lineEdit()->text();
         m_projectInfo.m_id = m_projectId->lineEdit()->text();
 
@@ -127,6 +128,7 @@ namespace O3DE::ProjectManager
         m_projectInfo = projectInfo;
 
         m_projectName->lineEdit()->setText(projectInfo.GetProjectDisplayName());
+        m_projectVersion->lineEdit()->setText(projectInfo.m_version);
         m_projectPath->lineEdit()->setText(projectInfo.m_path);
         m_projectId->lineEdit()->setText(projectInfo.m_id);
 
@@ -136,32 +138,61 @@ namespace O3DE::ProjectManager
         combobox->clear();
 
         // we use engine path which is unique instead of engine name which may not be
-        QString enginePath{};
+        EngineInfo assignedEngine;
         if(auto result = PythonBindingsInterface::Get()->GetProjectEngine(projectInfo.m_path); result)
         {
-            enginePath = result.GetValue<EngineInfo>().m_path;
+            assignedEngine = result.TakeValue();
         }
 
+        // handle case where user may not want to set the engine name (engine-centric) 
         int index = 0;
+        int selectedIndex = -1;
+        if (projectInfo.m_engineName.isEmpty() && !assignedEngine.m_path.isEmpty())
+        {
+            combobox->addItem(
+                QString("(no engine specified) %1 %2 (%3)").
+                    arg(assignedEngine.m_name,assignedEngine.m_version, assignedEngine.m_path),
+                    QStringList{ assignedEngine.m_path, "" });
+            selectedIndex = index;
+            index++;
+        }
+        // handle case when project uses an engine that isn't registered
+        else if (!projectInfo.m_engineName.isEmpty() && assignedEngine.m_path.isEmpty())
+        {
+            combobox->addItem(QString("%1 (not registered)").arg(projectInfo.m_engineName), QStringList{ "", projectInfo.m_engineName });
+            selectedIndex = index;
+            index++;
+        }
+
         if (auto result = PythonBindingsInterface::Get()->GetAllEngineInfos(); result)
         {
             for (auto engineInfo : result.GetValue<QVector<EngineInfo>>())
             {
                 if (!engineInfo.m_name.isEmpty())
                 {
+                    const bool useDisplayVersion = !engineInfo.m_displayVersion.isEmpty() &&
+                                                    engineInfo.m_displayVersion != "00.00" &&
+                                                    engineInfo.m_displayVersion != "0.1.0.0";
+                    const auto engineVersion = useDisplayVersion ? engineInfo.m_displayVersion : engineInfo.m_version;
+
                     combobox->addItem(
-                        QString("%1 (%2)").arg(engineInfo.m_name, engineInfo.m_path),
+                        QString("%1 %2 (%3)").arg(engineInfo.m_name, engineVersion, engineInfo.m_path),
                         QStringList{ engineInfo.m_path, engineInfo.m_name });
 
-                    if (!enginePath.isEmpty() && QDir(enginePath) == QDir(engineInfo.m_path))
+                    if (selectedIndex == -1 && !assignedEngine.m_path.isEmpty() && QDir(assignedEngine.m_path) == QDir(engineInfo.m_path))
                     {
-                        combobox->setCurrentIndex(index);
+                        selectedIndex = index;
                     }
                     index++;
                 }
             }
         }
 
+        if (selectedIndex != -1)
+        {
+            combobox->setCurrentIndex(selectedIndex);
+        }
+
         combobox->setVisible(combobox->count() > 0);
     }
 

scripts/o3de/o3de/compatibility.py

@@ -12,7 +12,6 @@ from packaging.version import Version, InvalidVersion
 from packaging.specifiers import SpecifierSet
 import pathlib
 import logging
-from collections import OrderedDict
 from o3de import manifest, utils, cmake, validation
 
 logger = logging.getLogger('o3de.compatibility')
@@ -54,17 +53,7 @@ def get_most_compatible_project_engine_path(project_path:pathlib.Path,
         return None
 
     if not engines_json_data:
-        engines_json_data = OrderedDict()
-        engines = manifest.get_manifest_engines()
-        for engine in engines:
-            if isinstance(engine, dict):
-                engine_path = pathlib.Path(engine['path']).resolve()
-            else:
-                engine_path = pathlib.Path(engine).resolve()
-            engine_json_data = manifest.get_engine_json_data(engine_path=engine_path)
-            if not engine_json_data:
-                continue
-            engines_json_data[engine_path] = engine_json_data
+        engines_json_data = manifest.get_engines_json_data_by_path()
 
     most_compatible_engine_path = None
     most_compatible_engine_version = None

scripts/o3de/o3de/manifest.py

@@ -404,6 +404,22 @@ def get_gem_templates(gem_path: pathlib.Path) -> list:
                         gem_object['templates'])) if 'templates' in gem_object else []
     return []
 
+def get_engines_json_data_by_path():
+    # dictionaries will maintain insertion order which we want
+    # because when we have engines with the same name and version
+    # we pick the first one found in the 'engines' o3de_manifest field
+    engines_json_data = {} 
+    engines = get_manifest_engines()
+    for engine in engines:
+        if isinstance(engine, dict):
+            engine_path = pathlib.Path(engine['path']).resolve()
+        else:
+            engine_path = pathlib.Path(engine).resolve()
+        engine_json_data = get_engine_json_data(engine_path=engine_path)
+        if not engine_json_data:
+            continue
+        engines_json_data[engine_path] = engine_json_data
+    return engines_json_data
 
 # Combined manifest queries
 def get_all_projects() -> list:

scripts/o3de/o3de/project_manager_interface.py

@@ -10,9 +10,8 @@ Contains functions for the project manager to call that gather data from o3de sc
 """
 
 import logging
-import pathlib
 
-from o3de import cmake, disable_gem, download, enable_gem, engine_properties, engine_template, manifest, project_properties, register, repo
+from o3de import manifest, utils
 
 logger = logging.getLogger('o3de.project_manager_interface')
 logging.basicConfig(format=utils.LOG_FORMAT)
@@ -138,7 +137,29 @@ def get_all_project_infos() -> list:
 
         :return list of dicts containing project infos.
     """
-    return list()
+    project_paths = manifest.get_all_projects()
+
+    # get all engine info once up front
+    engines_json_data = manifest.get_engines_json_data_by_path()
+
+    project_infos = []
+    for project_path in project_paths:
+        project_json_data = manifest.get_project_json_data(project_path=project_path)
+        if not project_json_data:
+            continue
+        user_project_json_data = manifest.get_project_json_data(project_path=project_path, user=True)
+        if user_project_json_data:
+            project_json_data.update(user_project_json_data)
+
+        project_json_data['path'] = project_path
+        project_json_data['engine_path'] = manifest.get_project_engine_path(project_path=project_path, 
+                                                                            project_json_data=project_json_data, 
+                                                                            user_project_json_data=user_project_json_data, 
+                                                                            engines_json_data=engines_json_data)
+        project_infos.append(project_json_data)
+    return project_infos
+
+
 
 
 def set_project_info(project_info: dict):