download missing remote files only

Canonical remote content will not appear in the projects screen if we don't download missing project objects.
Also, attempt to download all missing content on the remote repos page and, download missing gem content in Gem Catalog

Signed-off-by: Alex Peterson <[email protected]>

Code/Tools/ProjectManager/Source/GemRepo/GemRepoScreen.cpp

@@ -64,6 +64,8 @@ namespace O3DE::ProjectManager
 
     void GemRepoScreen::NotifyCurrentScreen()
     {
+        constexpr bool downloadMissingOnly = true;
+        PythonBindingsInterface::Get()->RefreshAllGemRepos(downloadMissingOnly);
         Reinit();
     }
 
@@ -171,7 +173,9 @@ namespace O3DE::ProjectManager
 
     void GemRepoScreen::HandleRefreshAllButton()
     {
-        bool refreshResult = PythonBindingsInterface::Get()->RefreshAllGemRepos();
+        // re-download everything when the user presses the refresh all button
+        constexpr bool downloadMissingOnly = false;
+        bool refreshResult = PythonBindingsInterface::Get()->RefreshAllGemRepos(downloadMissingOnly);
         Reinit();
         emit OnRefresh();
 
@@ -191,7 +195,9 @@ namespace O3DE::ProjectManager
         const QString repoUri = m_gemRepoModel->GetRepoUri(modelIndex);
         const QString repoName = m_gemRepoModel->GetName(modelIndex);
 
-        AZ::Outcome<void, AZStd::string> refreshResult = PythonBindingsInterface::Get()->RefreshGemRepo(repoUri);
+        // re-download everything when the user presses the refresh all button
+        constexpr bool downloadMissingOnly = false;
+        AZ::Outcome<void, AZStd::string> refreshResult = PythonBindingsInterface::Get()->RefreshGemRepo(repoUri, downloadMissingOnly);
         if (refreshResult.IsSuccess())
         {
             Reinit();

Code/Tools/ProjectManager/Source/PythonBindings.cpp

@@ -1581,14 +1581,14 @@ namespace O3DE::ProjectManager
         }
     }
 
-    AZ::Outcome<void, AZStd::string> PythonBindings::RefreshGemRepo(const QString& repoUri)
+    AZ::Outcome<void, AZStd::string> PythonBindings::RefreshGemRepo(const QString& repoUri, bool downloadMissingOnly)
     {
         bool refreshResult = false;
         AZ::Outcome<void, AZStd::string> result = ExecuteWithLockErrorHandling(
             [&]
             {
                 auto pyUri = QString_To_Py_String(repoUri);
-                auto pythonRefreshResult = m_repo.attr("refresh_repo")(pyUri);
+                auto pythonRefreshResult = m_repo.attr("refresh_repo")(pyUri, downloadMissingOnly);
 
                 // Returns an exit code so boolify it then invert result
                 refreshResult = !pythonRefreshResult.cast<bool>();
@@ -1606,13 +1606,13 @@ namespace O3DE::ProjectManager
         return AZ::Success();
     }
 
-    bool PythonBindings::RefreshAllGemRepos()
+    bool PythonBindings::RefreshAllGemRepos(bool downloadMissingOnly)
     {
         bool refreshResult = false;
         bool result = ExecuteWithLock(
             [&]
             {
-                auto pythonRefreshResult = m_repo.attr("refresh_repos")();
+                auto pythonRefreshResult = m_repo.attr("refresh_repos")(downloadMissingOnly);
 
                 // Returns an exit code so boolify it then invert result
                 refreshResult = !pythonRefreshResult.cast<bool>();

Code/Tools/ProjectManager/Source/PythonBindings.h

@@ -72,8 +72,8 @@ namespace O3DE::ProjectManager
         bool RemoveInvalidProjects() override;
 
         // Remote Repos
-        AZ::Outcome<void, AZStd::string> RefreshGemRepo(const QString& repoUri) override;
-        bool RefreshAllGemRepos() override;
+        AZ::Outcome<void, AZStd::string> RefreshGemRepo(const QString& repoUri, bool downloadMissingOnly = false) override;
+        bool RefreshAllGemRepos(bool downloadMissingOnly = false) override;
         DetailedOutcome AddGemRepo(const QString& repoUri) override;
         bool RemoveGemRepo(const QString& repoUri) override;
         bool SetRepoEnabled(const QString& repoUri, bool enabled) override;

Code/Tools/ProjectManager/Source/PythonBindingsInterface.h

@@ -306,15 +306,17 @@ namespace O3DE::ProjectManager
         /**
          * Refresh gem repo in the current engine.
          * @param repoUri the absolute filesystem path or url to the gem repo.
+         * @param downloadMissingOnly true to only download missing objects, if false, re-download everything
          * @return An outcome with the success flag as well as an error message in case of a failure.
          */
-        virtual AZ::Outcome<void, AZStd::string> RefreshGemRepo(const QString& repoUri) = 0;
+        virtual AZ::Outcome<void, AZStd::string> RefreshGemRepo(const QString& repoUri, bool downloadMissingOnly = false) = 0;
 
         /**
          * Refresh all gem repos in the current engine.
+         * @param downloadMissingOnly true to only download missing objects, if false, re-download everything
          * @return true on success, false on failure.
          */
-        virtual bool RefreshAllGemRepos() = 0;
+        virtual bool RefreshAllGemRepos(bool downloadMissingOnly = false) = 0;
 
         /**
          * Registers this gem repo with the current engine.

scripts/o3de/o3de/repo.py

@@ -64,7 +64,7 @@ def download_repo_manifest(manifest_uri: str, force_overwrite: bool = True) -> p
 
     return cache_file if result == 0 else None
 
-def download_object_manifests(repo_data):
+def download_object_manifests(repo_data: dict, download_missing_files_only: bool = False):
 
     if get_repo_schema_version(repo_data) == REPO_SCHEMA_VERSION_1_0_0:
         # schema version 1.0.0 includes all json data in repo.json
@@ -83,13 +83,14 @@ def download_object_manifests(repo_data):
             manifest_json_uri = f'{o3de_object_uri}/{manifest_json_filename}'
             cache_file, parsed_uri = get_cache_file_uri(manifest_json_uri)
 
-            git_provider = utils.get_git_provider(parsed_uri)
-            if git_provider:
-                parsed_uri = git_provider.get_specific_file_uri(parsed_uri)
+            if not cache_file.exists() or not download_missing_files_only:
+                git_provider = utils.get_git_provider(parsed_uri)
+                if git_provider:
+                    parsed_uri = git_provider.get_specific_file_uri(parsed_uri)
 
-            download_file_result = utils.download_file(parsed_uri, cache_file, True)
-            if download_file_result != 0:
-                return download_file_result
+                download_file_result = utils.download_file(parsed_uri, cache_file, True)
+                if download_file_result != 0:
+                    return download_file_result
     return 0
 
 def get_repo_schema_version(repo_data: dict):
@@ -184,7 +185,8 @@ def validate_remote_repo(repo_uri: str, validate_contained_objects: bool = False
     return True
 
 def process_add_o3de_repo(file_name: str or pathlib.Path,
-                          repo_set: set) -> int:
+                          repo_set: set,
+                          download_missing_files_only: bool = False) -> int:
     file_name = pathlib.Path(file_name).resolve()
     if not validation.valid_o3de_repo_json(file_name):
         logger.error(f'Repository JSON {file_name} could not be loaded or is missing required values')
@@ -211,7 +213,7 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
             logger.error(f'{file_name} failed to save: {str(e)}')
             return 1
 
-    if download_object_manifests(repo_data) != 0:
+    if download_object_manifests(repo_data, download_missing_files_only) != 0:
         return 1
 
     # Having a repo is also optional
@@ -223,11 +225,12 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
             repo_uri = f'{repo}/repo.json'
             cache_file, parsed_uri = get_cache_file_uri(repo_uri)
             
-            download_file_result = utils.download_file(parsed_uri, cache_file, True)
-            if download_file_result != 0:
-                return download_file_result
+            if not cache_file.is_file() or not download_missing_files_only:
+                download_file_result = utils.download_file(parsed_uri, cache_file, True)
+                if download_file_result != 0:
+                    return download_file_result
 
-            return process_add_o3de_repo(cache_file, repo_set)
+            return process_add_o3de_repo(cache_file, repo_set, download_missing_files_only)
     return 0
 
 def get_object_versions_json_data(remote_object_list:list, required_json_key:str = None, required_json_value:str = None) -> list:
@@ -261,8 +264,14 @@ def get_object_json_data_from_cached_repo(repo_uri: str, repo_key: str, object_t
 
     file_name = pathlib.Path(cache_file).resolve()
     if not file_name.is_file():
-        logger.error(f'Could not find cached repository json file for {repo_uri}. Try refreshing the repository.')
-        return list() 
+        logger.info(f'Could not find cached repository json file for {repo_uri}, attempting to download')
+
+        # attempt to download the missing repo.json
+        cache_file = download_repo_manifest(url)
+        file_name = pathlib.Path(cache_file).resolve()
+        if not file_name.is_file():
+            logger.error(f'Could not download the repository json file from {repo_uri}')
+            return list() 
 
     with file_name.open('r') as f:
         try:
@@ -289,13 +298,18 @@ def get_object_json_data_from_cached_repo(repo_uri: str, repo_key: str, object_t
                     manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
                     cache_object_json_filepath, _ = get_cache_file_uri(manifest_json_uri)
                     
-                    if cache_object_json_filepath.is_file():
-                        json_data = manifest.get_json_data_file(cache_object_json_filepath, object_typename, object_validator)
-                        # validation errors will be logged via the function above
-                        if json_data:
-                            o3de_object_json_data.append(json_data)
-                    else:
-                        logger.warning(f'Could not find cached {repo_key} json file {cache_object_json_filepath} for {o3de_object_uri} in repo {repo_uri}')
+                    if not cache_object_json_filepath.is_file():
+                        # attempt to download the missing file
+                        cache_object_json_filepath = download_repo_manifest(manifest_json_uri)
+                        if not cache_object_json_filepath:
+                            logger.warning(f'Could not download the missing cached {repo_key} json file {cache_object_json_filepath} from {manifest_json_uri} in repo {repo_uri}')
+                            continue
+
+                    json_data = manifest.get_json_data_file(cache_object_json_filepath, object_typename, object_validator)
+                    # validation errors will be logged via the function above
+                    if json_data:
+                        o3de_object_json_data.append(json_data)
+
         elif repo_schema_version == REPO_SCHEMA_VERSION_1_0_0:
             # the new schema version appends _data to the repo key
             # so it doesn't conflict with version 0.0.0 fields 
@@ -338,7 +352,8 @@ def get_template_json_data_from_all_cached_repos(enabled_only: bool = True) -> l
     return templates_json_data
 
 def refresh_repo(repo_uri: str,
-                 repo_set: set = None) -> int:
+                 repo_set: set = None,
+                 download_missing_files_only: bool = False) -> int:
 
     if not repo_uri_enabled(repo_uri):
         logger.info(f'Not refreshing {repo_uri} repo because it is deactivated.')
@@ -348,20 +363,21 @@ def refresh_repo(repo_uri: str,
         repo_set = set()
 
     repo_uri = f'{repo_uri}/repo.json'
-
-    cache_file = download_repo_manifest(repo_uri)
-    if not cache_file:
-        logger.error(f'Repo json {repo_uri} could not download.')
-        return 1
+    cache_file, _ = get_cache_file_uri(repo_uri)
+    if not cache_file.is_file() or not download_missing_files_only:
+        cache_file = download_repo_manifest(repo_uri)
+        if not cache_file:
+            logger.error(f'Repo json {repo_uri} could not download.')
+            return 1
 
     if not validation.valid_o3de_repo_json(cache_file):
         logger.error(f'Repo json {repo_uri} is not valid.')
         cache_file.unlink()
         return 1
 
-    return process_add_o3de_repo(cache_file, repo_set)
+    return process_add_o3de_repo(cache_file, repo_set, download_missing_files_only)
 
-def refresh_repos() -> int:
+def refresh_repos(download_missing_files_only: bool = False) -> int:
     result = 0
 
     # set will stop circular references
@@ -371,7 +387,7 @@ def refresh_repos() -> int:
         if repo_uri not in repo_set:
             repo_set.add(repo_uri)
 
-            last_failure = refresh_repo(repo_uri, repo_set)
+            last_failure = refresh_repo(repo_uri, repo_set, download_missing_files_only)
             if last_failure:
                 result = last_failure
 

scripts/o3de/tests/test_project_manager_interface.py

@@ -352,16 +352,28 @@ def test_edit_project_properties():
 # manifest interface
 def test_refresh_repo():
     sig = signature(repo.refresh_repo)
-    assert len(sig.parameters) >= 1
+    assert len(sig.parameters) >= 3
 
-    repo_uri = list(sig.parameters.values())[0]
-    assert repo_uri.name == 'repo_uri'
-    assert repo_uri.annotation == str
+    parameters = list(sig.parameters.values())
+
+    assert parameters[0].name == 'repo_uri'
+    assert parameters[0].annotation == str
+
+    assert parameters[1].name == 'repo_set'
+    assert parameters[1].annotation == set
+
+    assert parameters[2].name == 'download_missing_files_only'
+    assert parameters[2].annotation == bool
 
     assert sig.return_annotation == int
 
 def test_refresh_repos():
     sig = signature(repo.refresh_repos)
+    assert len(sig.parameters) >= 1
+    parameters = list(sig.parameters.values())
+
+    assert parameters[0].name == 'download_missing_files_only'
+    assert parameters[0].annotation == bool
 
     assert sig.return_annotation == int