Add support to register sub gem to nearest ancestor gem automatically (#15030)

* Removing __init__.py from the o3de package test folder

This __init__.py was inhibiting the pytest scanner from being able to
pick up import the tests folder.
Because there was a ` __init__.py` in the `<engine-root>/scripts/o3de/tests` it would cause an error in the [test_package_name](https://docs.pytest.org/en/latest/explanation/goodpractices.html#test-package-name) discovery when importing the o3de test modules.
They would attempt to be imported as `tests.test_<module>`, but the
issue is that the  `<engine-root>/scripts/o3de` directory is not part of
sys.path and therefore the import would fail.

```
______________ ERROR collecting scripts/o3de/tests/test_cmake.py
______________
ImportError while importing test module
'd:\o3de\o3de\scripts\o3de\tests\test_cmake.py'.
Hint: make sure your test modules/packages have valid Python names.
Traceback:
python\runtime\python-3.10.5-rev1-windows\python\lib\importlib\__init__.py:126:
in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
    E   ModuleNotFoundError: No module named 'tests.test_cmake'
```

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Add support to the o3de register command to automatically register gem
paths to the nearest ancestor gem directory if no explicitly
`--external-subdirectory-gem-path`,
`--external-subdirectory-project-path` nor
`--external-subdirectory-engine-path` argument was supplied.

If there is no ancestor gem path, then the nearest project path is
located.
If there is no nearest project pathm, then the nearest engine path is
located.
Finally if there is no nearest engine path, the
`~/.o3de/o3de_manifest.json` is where the gem will be registered.

Signed-off-by: lumberyard-employee-dm <[email protected]>

* Updated the register.py script with a command to force register with the
o3de_manifest.json

The `--force-register-with-o3de-manifest/-frwom` option can be used to
force registration of an external subdirectory with the users
`~/.o3de/o3de_manifest.json` file

Signed-off-by: lumberyard-employee-dm <[email protected]>

---------

Signed-off-by: lumberyard-employee-dm <[email protected]>

scripts/o3de/o3de/register.py

@@ -186,7 +186,7 @@ def register_all_projects_in_folder(projects_path: pathlib.Path,
                                     force: bool = False,
                                     dry_run: bool = False) -> int:
     return register_all_o3de_objects_of_type_in_folder(projects_path, 'project', remove, force,
-                                                       stop_on_template_folders, engine_path=engine_path, 
+                                                       stop_on_template_folders, engine_path=engine_path,
                                                        dry_run=dry_run)
 
 
@@ -396,22 +396,25 @@ def register_external_subdirectory(json_data: dict,
                                    remove: bool = False,
                                    engine_path: pathlib.Path = None,
                                    project_path: pathlib.Path = None,
-                                   gem_path: pathlib.Path = None) -> int:
+                                   gem_path: pathlib.Path = None,
+                                   force_register_with_o3de_manifest: bool = False) -> int:
     """
     :return An integer return code indicating whether registration or removal of the external subdirectory
     completed successfully
     """
     # If a gem path, project path or engine path has not been supplied auto detect which manifest to register the input path with
-    if not gem_path:
-        gem_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('gem.json'), external_subdir_path)
-    elif not project_path:
-        project_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('project.json'), external_subdir_path)
-    elif not engine_path:
-        engine_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('engine.json'), external_subdir_path)
+    # Start from the parent of the external_subdirectory to catch the case that if it is a gem, it does not register itself
+    if not force_register_with_o3de_manifest and not gem_path and not project_path and not engine_path:
+        if not gem_path and external_subdir_path.parent != external_subdir_path:
+            gem_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('gem.json'), external_subdir_path.parent)
+        if not gem_path:
+            project_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('project.json'), external_subdir_path)
+        if not project_path:
+            engine_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('engine.json'), external_subdir_path)
     return register_o3de_object_path(json_data, external_subdir_path, 'external_subdirectories', '', None, remove,
-                                     pathlib.Path(engine_path).resolve() if engine_path else None,
-                                     pathlib.Path(project_path).resolve() if project_path else None,
-                                     pathlib.Path(gem_path).resolve() if gem_path else None)
+                                     pathlib.Path(engine_path).resolve() if not force_register_with_o3de_manifest and engine_path else None,
+                                     pathlib.Path(project_path).resolve() if not force_register_with_o3de_manifest and project_path else None,
+                                     pathlib.Path(gem_path).resolve() if not force_register_with_o3de_manifest and gem_path else None)
 
 
 def register_gem_path(json_data: dict,
@@ -419,11 +422,19 @@ def register_gem_path(json_data: dict,
                       remove: bool = False,
                       engine_path: pathlib.Path = None,
                       project_path:  pathlib.Path = None,
+                      ancestor_gem_path: pathlib.Path = None,
                       force: bool = False,
-                      dry_run:bool = False) -> int:
-    # If a project path or engine path has not been supplied auto detect which manifest to register the input path with
-    if not project_path and not engine_path:
-        project_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('project.json'), gem_path)
+                      dry_run: bool = False,
+                      force_register_with_o3de_manifest: bool = False) -> int:
+    # If an ancestor gem_ path, project path or engine path has not been supplied
+    # auto detect which manifest to register the input path with
+
+    if not force_register_with_o3de_manifest and not ancestor_gem_path and not project_path and not engine_path:
+        # Start from the parent of the gem_path to make sure the gem doesn't register itself as an external subdirectory
+        if gem_path.parent != gem_path:
+            ancestor_gem_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('gem.json'), gem_path.parent)
+        if not ancestor_gem_path:
+            project_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('project.json'), gem_path)
         if not project_path:
             engine_path = utils.find_ancestor_dir_containing_file(pathlib.PurePath('engine.json'), gem_path)
 
@@ -433,12 +444,12 @@ def register_gem_path(json_data: dict,
             logger.error(f'Failed to load gem.json data needed for registration from {gem_path}')
             return 1
 
-        # do not check compatibility if the project has not been registered with an engine 
-        # because most gems depend on engine gems which would not be found 
+        # do not check compatibility if the project has not been registered with an engine
+        # because most gems depend on engine gems which would not be found
         if project_path and manifest.get_project_engine_path(project_path):
             # note this check includes engine and manifest gems
             incompatible_objects = compatibility.get_gem_project_incompatible_objects(gem_path, gem_json_data, project_path)
-            if incompatible_objects: 
+            if incompatible_objects:
                 logger.error(f'{gem_json_data["gem_name"]} is not known to be compatible with the '
                     'following objects/APIs and requires the --force parameter to register:\n  '+
                     "\n  ".join(incompatible_objects))
@@ -454,7 +465,7 @@ def register_gem_path(json_data: dict,
                 external_subdirectories=[gem_path])
 
             incompatible_objects = compatibility.get_gem_engine_incompatible_objects(gem_json_data, engine_json_data, engine_gems_json_data)
-            if incompatible_objects: 
+            if incompatible_objects:
                 logger.error(f'{gem_json_data["gem_name"]} is not known to be compatible with the '
                     'following objects/APIs and requires the --force parameter to register:\n  '+
                     "\n  ".join(incompatible_objects))
@@ -462,9 +473,10 @@ def register_gem_path(json_data: dict,
 
     result = register_o3de_object_path(json_data, gem_path, 'external_subdirectories', 'gem.json',
                                      validation.valid_o3de_gem_json, remove,
-                                     pathlib.Path(engine_path).resolve() if engine_path else None,
-                                     pathlib.Path(project_path).resolve() if project_path else None,
-                                     dry_run=dry_run)
+                                     pathlib.Path(engine_path).resolve() if not force_register_with_o3de_manifest and engine_path else None,
+                                     pathlib.Path(project_path).resolve() if not force_register_with_o3de_manifest and project_path else None,
+                                     pathlib.Path(ancestor_gem_path).resolve() if not force_register_with_o3de_manifest and ancestor_gem_path else None,
+                                     dry_run=dry_run,)
 
     if result == 0 and dry_run:
         logger.info(f'Gem path {gem_path} was not registered because the --dry-run option was specified')
@@ -781,7 +793,8 @@ def register(engine_path: pathlib.Path = None,
              external_subdir_gem_path: pathlib.Path = None,
              remove: bool = False,
              force: bool = False,
-             dry_run: bool = False
+             dry_run: bool = False,
+             force_register_with_o3de_manifest: bool = False
              ) -> int:
     """
     Adds/Updates entries to the ~/.o3de/o3de_manifest.json
@@ -839,15 +852,18 @@ def register(engine_path: pathlib.Path = None,
             return 1
         result = result or register_gem_path(json_data, gem_path, remove,
                                              external_subdir_engine_path, external_subdir_project_path,
-                                             force, dry_run)
+                                             external_subdir_gem_path,
+                                             force, dry_run,
+                                             force_register_with_o3de_manifest=force_register_with_o3de_manifest)
 
     if isinstance(external_subdir_path, pathlib.PurePath):
         if not external_subdir_path:
             logger.error(f'External Subdirectory path is None.')
             return 1
         result = result or register_external_subdirectory(json_data, external_subdir_path, remove,
-                                                          external_subdir_engine_path, external_subdir_project_path, 
-                                                          external_subdir_gem_path)
+                                                          external_subdir_engine_path, external_subdir_project_path,
+                                                          external_subdir_gem_path,
+                                                          force_register_with_o3de_manifest=force_register_with_o3de_manifest)
 
     if isinstance(template_path, pathlib.PurePath):
         if not template_path:
@@ -938,7 +954,8 @@ def _run_register(args: argparse) -> int:
                         external_subdir_gem_path=args.external_subdirectory_gem_path,
                         remove=args.remove,
                         force=args.force,
-                        dry_run=args.dry_run)
+                        dry_run=args.dry_run,
+                        force_register_with_o3de_manifest=args.force_register_with_o3de_manifest)
 
 
 def add_parser_args(parser):
@@ -1015,6 +1032,8 @@ def add_parser_args(parser):
                                             ' the engine-path location')
     external_subdir_path_group.add_argument('-espp', '--external-subdirectory-project-path', type=pathlib.Path)
     external_subdir_path_group.add_argument('-esgp', '--external-subdirectory-gem-path', type=pathlib.Path,  help='If supplied, registers the external subdirectory with the gem.json at the gem-path location')
+    external_subdir_path_group.add_argument('-frwom', '--force-register-with-o3de-manifest', action='store_true', default=False,
+                                            help='When set, forces the registration of the external subdirectory with the ~/.o3de/o3de_manifest.json')
     parser.set_defaults(func=_run_register)
 
 

scripts/o3de/tests/__init__.py

@@ -1,7 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#

scripts/o3de/tests/test_register.py

@@ -22,7 +22,7 @@ string_manifest_data = '{}'
         pytest.param(pathlib.PurePath('D:/o3de/o3de'), "o3de", False, 0),
         # Same engine_name and path should result in valid registration
         pytest.param(pathlib.PurePath('D:/o3de/o3de'), "o3de", False, 0),
-        # Same engine_name but different path succeeds 
+        # Same engine_name but different path succeeds
         pytest.param(pathlib.PurePath('D:/o3de/engine-path'), "o3de", False, 0),
         # New engine_name should result in valid registration
         pytest.param(pathlib.PurePath('D:/o3de/engine-path'), "o3de-other", False, 0),
@@ -138,6 +138,26 @@ TEST_GEM_JSON_PAYLOAD = '''
 }
 '''
 
+TEST_SUB_GEM_JSON_PAYLOAD = '''
+{
+    "gem_name": "TestSubGem",
+    "version": "0.0.0",
+    "display_name": "TestSubGem",
+    "license": "What license TestGem uses goes here: i.e. https://opensource.org/licenses/MIT",
+    "origin": "The primary repo for TestSubGem goes here: i.e. http://www.mydomain.com",
+    "type": "Code",
+    "summary": "A short description of TestSubGem.",
+    "canonical_tags": [
+        "Gem"
+    ],
+    "user_tags": [
+        "TestSubGem"
+    ],
+    "icon_path": "preview.png",
+    "requirements": ""
+}
+'''
+
 TEST_PROJECT_JSON_PAYLOAD = '''
 {
     "project_name": "TestProject",
@@ -180,29 +200,43 @@ def init_register_gem_data(request):
     request.cls.o3de_manifest_data = json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD)
     request.cls.project_data = json.loads(TEST_PROJECT_JSON_PAYLOAD)
     request.cls.engine_data = json.loads(TEST_ENGINE_JSON_PAYLOAD)
+    request.cls.ancestor_gem_data = json.loads(TEST_GEM_JSON_PAYLOAD)
 
 
 @pytest.mark.usefixtures('init_register_gem_data')
 class TestRegisterGem:
     engine_path = pathlib.PurePath('o3de')
     project_path = pathlib.PurePath('TestProject')
+    ancestor_gem_path = pathlib.PurePath('TestGem')
 
     @staticmethod
     def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None,
                         project_path: pathlib.Path = None) -> dict or None:
-        return json.loads(TEST_GEM_JSON_PAYLOAD)
-
-    @pytest.mark.parametrize("gem_path, expected_manifest_file, dry_run, expected_result", [
-                                 pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('o3de_manifest.json'), False, 0),
-                                 pytest.param(project_path / 'TestGem', pathlib.PurePath('project.json'), False, 0),
-                                 pytest.param(engine_path / 'TestGem', pathlib.PurePath('engine.json'), False, 0),
-                                 pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('o3de_manifest.json'), True, 0),
-                                 pytest.param(project_path / 'TestGem', pathlib.PurePath('project.json'), True, 0),
-                                 pytest.param(engine_path / 'TestGem', pathlib.PurePath('engine.json'), True, 0),
+        if (gem_name and gem_name == "TestSubGem") or \
+            (gem_path and gem_path.name == "TestSubGem"):
+            return json.loads(TEST_SUB_GEM_JSON_PAYLOAD)
+        else:
+            return json.loads(TEST_GEM_JSON_PAYLOAD)
+
+    @pytest.mark.parametrize("gem_path, expected_manifest_file, dry_run, force_o3de_manifest_register, expected_result", [
+                                 pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('o3de_manifest.json'), False, False, 0),
+                                 pytest.param(project_path / 'TestGem', pathlib.PurePath('project.json'), False, False, 0),
+                                 pytest.param(project_path / 'TestGem', pathlib.PurePath('o3de_manifest.json'), False, True, 0),
+                                 pytest.param(engine_path / 'TestGem', pathlib.PurePath('engine.json'), False, False, 0),
+                                 pytest.param(engine_path / 'TestGem', pathlib.PurePath('o3de_manifest.json'), False, True, 0),
+                                 pytest.param(pathlib.PurePath('TestGem/TestSubGem') , pathlib.PurePath('gem.json'), False, False, 0),
+                                 pytest.param(pathlib.PurePath('TestGem/TestSubGem') , pathlib.PurePath('o3de_manifest.json'), False, True, 0),
+                                 pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('o3de_manifest.json'), True, False, 0),
+                                 pytest.param(project_path / 'TestGem', pathlib.PurePath('project.json'), True, False, 0),
+                                 pytest.param(engine_path / 'TestGem', pathlib.PurePath('engine.json'), True, False, 0),
+                                 pytest.param(pathlib.PurePath('TestGem/TestSubGem') , pathlib.PurePath('gem.json'), True, False, 0),
                              ])
-    def test_register_gem_auto_detects_manifest_update(self, gem_path, expected_manifest_file, dry_run, expected_result):
+    def test_register_gem_auto_detects_manifest_update(self, gem_path, expected_manifest_file, dry_run,
+     force_o3de_manifest_register, expected_result):
 
         def save_o3de_manifest(manifest_data: dict, manifest_path: pathlib.Path = None) -> bool:
+            if manifest_path == pathlib.Path(TestRegisterGem.ancestor_gem_path).resolve() / 'gem.json':
+                self.ancestor_gem_data = manifest_data
             if manifest_path == pathlib.Path(TestRegisterGem.project_path).resolve() / 'project.json':
                 self.project_data = manifest_data
             elif manifest_path == pathlib.Path(TestRegisterGem.engine_path).resolve() / 'engine.json':
@@ -212,6 +246,8 @@ class TestRegisterGem:
             return True
 
         def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict:
+            if manifest_path == TestRegisterGem.ancestor_gem_path:
+                return self.ancestor_gem_data
             if manifest_path == TestRegisterGem.project_path:
                 return self.project_data
             elif manifest_path == TestRegisterGem.engine_path:
@@ -227,6 +263,12 @@ class TestRegisterGem:
             return json.loads(TEST_PROJECT_JSON_PAYLOAD)
 
         def find_ancestor_dir(target_file_name: pathlib.PurePath, start_path: pathlib.Path):
+            try:
+                if target_file_name == pathlib.PurePath('gem.json')\
+                        and start_path.relative_to(TestRegisterGem.ancestor_gem_path):
+                    return TestRegisterGem.ancestor_gem_path
+            except ValueError:
+                pass
             try:
                 if target_file_name == pathlib.PurePath('project.json')\
                         and start_path.relative_to(TestRegisterGem.project_path):
@@ -249,19 +291,23 @@ class TestRegisterGem:
                 patch('o3de.utils.find_ancestor_dir_containing_file', side_effect=find_ancestor_dir) as _6,\
                 patch('pathlib.Path.is_dir', return_value=True) as _7,\
                 patch('o3de.validation.valid_o3de_gem_json', return_value=True) as _8:
-            result = register.register(gem_path=gem_path, dry_run=dry_run)
+            result = register.register(gem_path=gem_path, dry_run=dry_run,
+                force_register_with_o3de_manifest=force_o3de_manifest_register)
             assert result == expected_result
 
             if expected_manifest_file == pathlib.PurePath('o3de_manifest.json'):
                 external_subdirectories = map(lambda subdir: pathlib.PurePath(subdir),
                                         self.o3de_manifest_data.get('external_subdirectories', []))
+            elif expected_manifest_file == pathlib.PurePath('gem.json'):
+                external_subdirectories = map(lambda subdir: TestRegisterGem.ancestor_gem_path / subdir,
+                                       self.ancestor_gem_data.get('external_subdirectories', []))
             elif expected_manifest_file == pathlib.PurePath('project.json'):
                 external_subdirectories = map(lambda subdir: TestRegisterGem.project_path / subdir,
                                        self.project_data.get('external_subdirectories', []))
             elif expected_manifest_file == pathlib.PurePath('engine.json'):
                 external_subdirectories = map(lambda subdir: TestRegisterGem.engine_path / subdir,
                                        self.engine_data.get('external_subdirectories', []))
-            
+
             gem_path = pathlib.Path(gem_path).resolve() if expected_manifest_file == pathlib.PurePath('o3de_manifest.json') else gem_path
             if dry_run:
                 assert gem_path not in external_subdirectories
@@ -299,9 +345,9 @@ class TestRegisterProject:
             # passes when compatible_engines has match
             pytest.param('o3de7', '0.0.0', None, { 'gem1':'' }, None, None, ['gem1'], ['o3de7'], None, None, None, False, False, 0),
             pytest.param('o3de8', '1.2.3', None, { 'gem1':'' }, None, None, ['gem1'], ['o3de8>=1.2.3','o3de-sdk==2.3.4'], None, None, None, False, False, 0),
-            # fails when gem is used that is not known to be compatible with version 1.2.3 
+            # fails when gem is used that is not known to be compatible with version 1.2.3
             pytest.param('o3de9', '1.2.3', None, { 'gem1':'' }, None, None, ['gem1'], ['o3de9'], None, ['o3de==2.3.4'], None, False, False, 1),
-            # passes when gem is used that is known compatible with version 1.2.3 
+            # passes when gem is used that is known compatible with version 1.2.3
             pytest.param('o3de10', '1.2.3', None, { 'gem1':'' }, None, None, ['gem1'], ['o3de10'], None, ['o3de10==1.2.3'], None, False, False, 0),
             # passes when compatible engine not found but compatible api found
             pytest.param('o3de11', '1.2.3', {'api':'1.2.3'}, { 'gem1':'' }, "", "", ['gem1'], ['o3de11==2.3.4'], ['api==1.2.3'], None, None, False, False, 0),
@@ -318,7 +364,7 @@ class TestRegisterProject:
         ]
     )
     def test_register_project(self, test_engine_name, engine_version, engine_api_versions,
-                                registered_gem_versions, project_engine_name, project_engine_version, 
+                                registered_gem_versions, project_engine_name, project_engine_version,
                                 project_gems, project_compatible_engines, project_engine_api_dependencies,
                                 gem_compatible_engines, gem_engine_api_dependencies,
                                 force, dry_run, expected_result):
@@ -338,8 +384,8 @@ class TestRegisterProject:
                 return self.engine_data
             return self.o3de_manifest_data
 
-        def get_gems_json_data_by_name( engine_path:pathlib.Path = None, 
-                                        project_path: pathlib.Path = None, 
+        def get_gems_json_data_by_name( engine_path:pathlib.Path = None,
+                                        project_path: pathlib.Path = None,
                                         include_manifest_gems: bool = False,
                                         include_engine_gems: bool = False,
                                         external_subdirectories: list = None
@@ -370,7 +416,7 @@ class TestRegisterProject:
             if test_engine_name != None:
                 engine_json_data['engine_name'] = test_engine_name
 
-            # we want to allow for testing the case where these fields 
+            # we want to allow for testing the case where these fields
             # are missing or empty
             if engine_version != None:
                 engine_json_data['version'] = engine_version
@@ -384,7 +430,7 @@ class TestRegisterProject:
                                 user: bool = False) -> dict or None:
             project_json_data = json.loads(TEST_PROJECT_JSON_PAYLOAD)
 
-            # we want to allow for testing the case where these fields 
+            # we want to allow for testing the case where these fields
             # are missing or empty
             if project_engine_name != None:
                 project_json_data['engine'] = project_engine_name
@@ -444,5 +490,5 @@ class TestRegisterProject:
             if dry_run:
                 assert TestRegisterProject.project_path not in map(lambda subdir: pathlib.PurePath(subdir),
                                         self.o3de_manifest_data.get('external_subdirectories', []))
-                 
+
             assert result == expected_result