Add verbosity option to some o3de.py sub-commands (#6079)

* Add verbosity option to some o3de.py sub-commands

- Added the -v/-vv option to the most commonly used o3de sub-commands
- Adds a custom argparse Action to make it easy to add verbosity option
  to other commands as needed.
- Sets up parent-child loggers so that when verbosity is changed in a
  sub-command it will be applied globally to all logger instances.

Also:
- Fixes up many warnings, PEP8 improvements

Signed-off-by: amzn-phist <[email protected]>

* Update the log format in o3de script modules

- Use a bit nicer format than the default, which was harder to read.

Signed-off-by: amzn-phist <[email protected]>

scripts/o3de.py

@@ -7,20 +7,25 @@
 #
 
 import argparse
+import logging
 import pathlib
 import sys
 
+logger = logging.getLogger('o3de')
 
-def add_args(parser, subparsers) -> None:
+
+def add_args(parser: argparse.ArgumentParser) -> None:
     """
     add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be
     invoked by o3de.py
     Ex o3de.py can invoke the register  downloadable commands by importing register,
     call add_args and execute: python o3de.py register --gem-path "C:/TestGem"
-    :param parser: the caller instantiates a parser and passes it in here
-    :param subparsers: the caller instantiates subparsers and passes it in here
+    :param parser: the caller instantiates an ArgumentParser and passes it in here
     """
 
+    subparsers = parser.add_subparsers(help='To get help on a sub-command:\no3de.py <sub-command> -h',
+                                       title='Sub-Commands')
+
     # As o3de.py shares the same name as the o3de package attempting to use a regular
     # from o3de import <module> line tries to import from the current o3de.py script and not the package
     # So the {current script directory} / 'o3de' is added to the front of the sys.path
@@ -28,24 +33,25 @@ def add_args(parser, subparsers) -> None:
     o3de_package_dir = (script_dir / 'o3de').resolve()
     # add the scripts/o3de directory to the front of the sys.path
     sys.path.insert(0, str(o3de_package_dir))
-    from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \
+    from o3de import engine_properties, engine_template, gem_properties, \
+        global_project, register, print_registration, get_registration, \
         enable_gem, disable_gem, project_properties, sha256, download
     # Remove the temporarily added path
     sys.path = sys.path[1:]
 
-    # global_project
+    # global project
     global_project.add_args(subparsers)
 
-    # engine templaate
+    # engine template
     engine_template.add_args(subparsers)
 
-    # register
+    # registration
     register.add_args(subparsers)
 
-    # show
+    # show registration
     print_registration.add_args(subparsers)
 
-    # get-registered
+    # get registration
     get_registration.add_args(subparsers)
 
     # add a gem to a project
@@ -74,11 +80,8 @@ if __name__ == "__main__":
     # parse the command line args
     the_parser = argparse.ArgumentParser()
 
-    # add subparsers
-    the_subparsers = the_parser.add_subparsers(help='sub-command help')
-
     # add args to the parser
-    add_args(the_parser, the_subparsers)
+    add_args(the_parser)
 
     # parse args
     the_args = the_parser.parse_args()
@@ -89,7 +92,8 @@ if __name__ == "__main__":
         sys.exit(1)
 
     # run
-    ret = the_args.func(the_args)
+    ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
+    logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
 
     # return
     sys.exit(ret)

scripts/o3de/o3de/cmake.py

@@ -13,10 +13,10 @@ import logging
 import os
 import pathlib
 
-from o3de import manifest
+from o3de import manifest, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.cmake')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 enable_gem_start_marker = 'set(ENABLED_GEMS'
 enable_gem_end_marker = ')'

scripts/o3de/o3de/disable_gem.py

@@ -15,10 +15,10 @@ import os
 import pathlib
 import sys
 
-from o3de import cmake, manifest
+from o3de import cmake, manifest, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.disable_gem')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def disable_gem_in_project(gem_name: str = None,
@@ -73,7 +73,6 @@ def disable_gem_in_project(gem_name: str = None,
         logger.error(f'Gem Path {gem_path} does not exist.')
         return 1
 
-
     # Read gem.json from the gem path
     gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
     if not gem_json_data:
@@ -116,6 +115,10 @@ def add_parser_args(parser):
     Ex. Directly run from this file alone with: python disable_gem.py --project-path D:/Test --gem-name Atom
     :param parser: the caller passes an argparse parser like instance to this method
     """
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(parser)
+
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
                        help='The path to the project.')
@@ -155,8 +158,6 @@ def main():
     # parse the command line args
     the_parser = argparse.ArgumentParser()
 
-    # add subparsers
-
     # add args to the parser
     add_parser_args(the_parser)
 
@@ -165,6 +166,7 @@ def main():
 
     # run
     ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
+    logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
 
     # return
     sys.exit(ret)

scripts/o3de/o3de/download.py

@@ -24,8 +24,9 @@ from datetime import datetime
 
 from o3de import manifest, repo, utils, validation, register
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.download')
+logging.basicConfig(format=utils.LOG_FORMAT)
+
 
 def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str) -> dict:
     json_data = {}
@@ -38,6 +39,7 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str
 
     return json_data
 
+
 def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path,
                                    manifest_json_name) -> int:
     # if the json has a sha256 check it against a sha256 of the zip

scripts/o3de/o3de/enable_gem.py

@@ -16,10 +16,10 @@ import os
 import pathlib
 import sys
 
-from o3de import cmake, manifest, register, validation
+from o3de import cmake, manifest, register, validation, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.enable_gem')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def enable_gem_in_project(gem_name: str = None,
@@ -132,6 +132,10 @@ def add_parser_args(parser):
     Ex. Directly run from this file alone with: python enable_gem.py --project-path "D:/TestProject" --gem-path "D:/TestGem"
     :param parser: the caller passes an argparse parser like instance to this method
     """
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(parser)
+
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
                        help='The path to the project.')
@@ -171,8 +175,6 @@ def main():
     # parse the command line args
     the_parser = argparse.ArgumentParser()
 
-    # add subparsers
-
     # add args to the parser
     add_parser_args(the_parser)
 
@@ -181,6 +183,7 @@ def main():
 
     # run
     ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
+    logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
 
     # return
     sys.exit(ret)

scripts/o3de/o3de/engine_properties.py

@@ -15,8 +15,9 @@ import logging
 
 from o3de import manifest, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.engine_properties')
+logging.basicConfig(format=utils.LOG_FORMAT)
+
 
 def edit_engine_props(engine_path: pathlib.Path = None,
                       engine_name: str = None,

scripts/o3de/o3de/engine_template.py

@@ -20,8 +20,8 @@ import re
 
 from o3de import manifest, register, validation, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.engine_template')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 binary_file_ext = {
     '.pak',
@@ -86,6 +86,7 @@ restricted_platforms = {
 template_file_name = 'template.json'
 this_script_parent = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
 
+
 def _replace_license_text(source_data: str):
     while '{BEGIN_LICENSE}' in source_data:
         start = source_data.find('{BEGIN_LICENSE}')
@@ -181,7 +182,7 @@ def _execute_template_json(json_data: dict,
     # regular copy if not templated
     for copy_file in json_data['copyFiles']:
         # construct the input file name
-        in_file = template_path / 'Template' /copy_file['file']
+        in_file = template_path / 'Template' / copy_file['file']
 
         # the file can be marked as optional, if it is and it does not exist skip
         if copy_file['isOptional'] and copy_file['isOptional'] == 'true':
@@ -246,7 +247,7 @@ def _execute_restricted_template_json(json_data: dict,
     for copy_file in json_data['copyFiles']:
         # construct the input file name
         in_file = template_restricted_path / restricted_platform / template_restricted_platform_relative_path\
-                  / template_name / 'Template'/ copy_file['file']
+                  / template_name / 'Template' / copy_file['file']
 
         # the file can be marked as optional, if it is and it does not exist skip
         if copy_file['isOptional'] and copy_file['isOptional'] == 'true':
@@ -364,7 +365,7 @@ def create_template(source_path: pathlib.Path,
                     keep_restricted_in_template: bool = False,
                     keep_license_text: bool = False,
                     replace: list = None,
-                    force: bool = False)  -> int:
+                    force: bool = False) -> int:
     """
     Create a template from a source directory using replacement
 
@@ -1205,12 +1206,12 @@ def create_from_template(destination_path: pathlib.Path,
                 # something is wrong with either the --template-restricted-platform-relative or the template is.
                 if template_restricted_platform_relative_path != template_json_restricted_platform_relative_path:
                     logger.error(f'The supplied --template-restricted-platform-relative-path'
-                                f' "{template_restricted_platform_relative_path}" does not match the'
-                                f' templates.json  "restricted_platform_relative_path". Either'
-                                f' --template-restricted-platform-relative-path is incorrect or the templates'
-                                f' "restricted_platform_relative_path" is wrong. Note that since this template'
-                                f' specifies "restricted_platform_relative_path" it need not be supplied and'
-                                f' "{template_json_restricted_platform_relative_path}" will be used.')
+                                 f' "{template_restricted_platform_relative_path}" does not match the'
+                                 f' templates.json  "restricted_platform_relative_path". Either'
+                                 f' --template-restricted-platform-relative-path is incorrect or the templates'
+                                 f' "restricted_platform_relative_path" is wrong. Note that since this template'
+                                 f' specifies "restricted_platform_relative_path" it need not be supplied and'
+                                 f' "{template_json_restricted_platform_relative_path}" will be used.')
                     return 1
     else:
         # The user has not supplied --template-restricted-platform-relative-path, try to read it from
@@ -1306,7 +1307,7 @@ def create_from_template(destination_path: pathlib.Path,
             os.makedirs(destination_restricted_path, exist_ok=True)
 
             # read the restricted_name from the destination restricted.json
-            restricted_json = destination_restricted_path / restricted.json
+            restricted_json = destination_restricted_path / 'restricted.json'
             if not os.path.isfile(restricted_json):
                 with open(restricted_json, 'w') as s:
                     restricted_json_data = {}
@@ -1956,7 +1957,6 @@ def create_gem(gem_path: pathlib.Path,
     replacements.append(("${NameLower}", gem_name.lower()))
     replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
 
-
     # module id is a uuid with { and -
     if module_id:
         replacements.append(("${ModuleClassId}", module_id))
@@ -2157,8 +2157,13 @@ def add_args(subparsers) -> None:
     call add_args and execute: python o3de.py create-gem --gem-path TestGem
     :param subparsers: the caller instantiates subparsers and passes it in here
     """
+
     # turn a directory into a template
     create_template_subparser = subparsers.add_parser('create-template')
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(create_template_subparser)
+
     create_template_subparser.add_argument('-sp', '--source-path', type=pathlib.Path, required=True,
                                            help='The path to the source that you want to make into a template')
     create_template_subparser.add_argument('-tp', '--template-path', type=pathlib.Path, required=False,
@@ -2233,6 +2238,10 @@ def add_args(subparsers) -> None:
 
     # create from template
     create_from_template_subparser = subparsers.add_parser('create-from-template')
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(create_from_template_subparser)
+
     create_from_template_subparser.add_argument('-dp', '--destination-path', type=pathlib.Path, required=True,
                                                 help='The path to where you want the template instantiated,'
                                                      ' can be absolute or relative to the current working directory.'
@@ -2316,6 +2325,10 @@ def add_args(subparsers) -> None:
 
     # creation of a project from a template (like create from template but makes project assumptions)
     create_project_subparser = subparsers.add_parser('create-project')
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(create_project_subparser)
+
     create_project_subparser.add_argument('-pp', '--project-path', type=pathlib.Path, required=True,
                                           help='The location of the project you wish to create from the template,'
                                                ' can be an absolute path or relative to the current working directory.'
@@ -2330,7 +2343,7 @@ def add_args(subparsers) -> None:
     group = create_project_subparser.add_mutually_exclusive_group(required=False)
     group.add_argument('-tp', '--template-path', type=pathlib.Path, required=False,
                        default=None,
-                       help='the path to the template you want to instance, can be absolute or'
+                       help='The path to the template you want to instance, can be absolute or'
                             ' relative to default templates path')
     group.add_argument('-tn', '--template-name', type=str, required=False,
                        default=None,
@@ -2340,7 +2353,7 @@ def add_args(subparsers) -> None:
     group = create_project_subparser.add_mutually_exclusive_group(required=False)
     group.add_argument('-prp', '--project-restricted-path', type=pathlib.Path, required=False,
                        default=None,
-                       help='path to the projects restricted folder, can be absolute or relative to'
+                       help='The path to the projects restricted folder, can be absolute or relative to'
                             ' the default restricted projects directory')
     group.add_argument('-prn', '--project-restricted-name', type=str, required=False,
                        default=None,
@@ -2351,7 +2364,7 @@ def add_args(subparsers) -> None:
     group.add_argument('-trp', '--template-restricted-path', type=pathlib.Path, required=False,
                        default=None,
                        help='The templates restricted path can be absolute or relative to'
-                             'the default restricted templates directory')
+                            ' the default restricted templates directory')
     group.add_argument('-trn', '--template-restricted-name', type=str, required=False,
                        default=None,
                        help='The name of the registered templates restricted path. If supplied this will resolve'
@@ -2413,6 +2426,10 @@ def add_args(subparsers) -> None:
 
     # creation of a gem from a template (like create from template but makes gem assumptions)
     create_gem_subparser = subparsers.add_parser('create-gem')
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(create_gem_subparser)
+
     create_gem_subparser.add_argument('-gp', '--gem-path', type=pathlib.Path, required=True,
                                       help='The gem path, can be absolute or relative to the current working directory')
     create_gem_subparser.add_argument('-gn', '--gem-name', type=str,
@@ -2512,16 +2529,18 @@ if __name__ == "__main__":
     the_parser = argparse.ArgumentParser()
 
     # add subparsers
-    the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True)
+    subparsers = the_parser.add_subparsers(help='To get help on a sub-command:\nengine_template.py <sub-command> -h',
+                                           title='Sub-Commands', dest='command', required=True)
 
-    # add args to the parser
-    add_args(the_subparsers)
+    # add args to the parsers
+    add_args(subparsers)
 
     # parse args
     the_args = the_parser.parse_args()
 
     # run
     ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
+    logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
 
     # return
     sys.exit(ret)

scripts/o3de/o3de/gem_properties.py

@@ -15,8 +15,8 @@ import logging
 
 from o3de import manifest, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.gem_properties')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def update_values_in_key_list(existing_values: list, new_values: list or str, remove_values: list or str,

scripts/o3de/o3de/get_registration.py

@@ -12,17 +12,18 @@ import sys
 
 from o3de import manifest
 
+
 def _run_get_registered(args: argparse) -> str or pathlib.Path:
     if args.override_home_folder:
         manifest.override_home_folder = args.override_home_folder
 
     return manifest.get_registered(args.engine_name,
-                          args.project_name,
-                          args.gem_name,
-                          args.template_name,
-                          args.default_folder,
-                          args.repo_name,
-                          args.restricted_name)
+                                   args.project_name,
+                                   args.gem_name,
+                                   args.template_name,
+                                   args.default_folder,
+                                   args.repo_name,
+                                   args.restricted_name)
 
 
 def add_parser_args(parser):

scripts/o3de/o3de/global_project.py

@@ -14,14 +14,15 @@ import re
 import pathlib
 import json
 
-from o3de import manifest, validation
+from o3de import manifest, validation, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.global_project')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 DEFAULT_BOOTSTRAP_SETREG = pathlib.Path('~/.o3de/Registry/bootstrap.setreg').expanduser()
 PROJECT_PATH_KEY = ('Amazon', 'AzCore', 'Bootstrap', 'project_path')
 
+
 def get_json_data(input_path: pathlib.Path):
     setreg_json_data = {}
     # If the output_path exist validate that it is a valid json file

scripts/o3de/o3de/manifest.py

@@ -18,8 +18,8 @@ import hashlib
 
 from o3de import validation, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.manifest')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 # Directory methods
 override_home_folder = None
@@ -41,6 +41,7 @@ def get_o3de_folder() -> pathlib.Path:
     o3de_folder.mkdir(parents=True, exist_ok=True)
     return o3de_folder
 
+
 def get_o3de_user_folder() -> pathlib.Path:
     o3de_user_folder = get_home_folder() / 'O3DE'
     o3de_user_folder.mkdir(parents=True, exist_ok=True)
@@ -177,6 +178,7 @@ def get_default_o3de_manifest_json_data() -> dict:
 
     return json_data
 
+
 def get_o3de_manifest() -> pathlib.Path:
     manifest_path = get_o3de_folder() / 'o3de_manifest.json'
     if not manifest_path.is_file():
@@ -228,11 +230,11 @@ def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> b
 
 
 def get_gems_from_subdirectories(external_subdirs: list) -> list:
-    '''
+    """
     Helper Method for scanning a set of external subdirectories for gem.json files
-    '''
+    """
     def is_gem_subdirectory(subdir_files):
-        for name in files:
+        for name in subdir_files:
             if name == 'gem.json':
                 return True
         return False
@@ -285,13 +287,14 @@ def get_repos() -> list:
     json_data = load_o3de_manifest()
     return json_data['repos'] if 'repos' in json_data else []
 
+
 # engine.json queries
 def get_engine_projects() -> list:
     engine_path = get_this_engine_path()
     engine_object = get_engine_json_data(engine_path=engine_path)
     if engine_object:
         return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(),
-                          engine_object['projects'])) if 'projects' in engine_object else []
+                        engine_object['projects'])) if 'projects' in engine_object else []
     return []
 
 
@@ -313,7 +316,7 @@ def get_engine_templates() -> list:
     engine_object = get_engine_json_data(engine_path=engine_path)
     if engine_object:
         return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(),
-                          engine_object['templates'])) if 'templates' in engine_object else []
+                        engine_object['templates'])) if 'templates' in engine_object else []
     return []
 
 
@@ -335,7 +338,7 @@ def get_project_external_subdirectories(project_path: pathlib.Path) -> list:
     project_object = get_project_json_data(project_path=project_path)
     if project_object:
         return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(),
-                   project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else []
+                        project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else []
     return []
 
 
@@ -343,7 +346,7 @@ def get_project_templates(project_path: pathlib.Path) -> list:
     project_object = get_project_json_data(project_path=project_path)
     if project_object:
         return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(),
-                          project_object['templates'])) if 'templates' in project_object else []
+                        project_object['templates'])) if 'templates' in project_object else []
     return []
 
 
@@ -433,6 +436,7 @@ def get_templates_for_generic_creation():  # temporary until we have a better wa
 
     return list(filter(filter_project_and_gem_templates_out, get_all_templates()))
 
+
 def get_json_file_path(object_typename: str,
                        object_path: str or pathlib.Path) -> pathlib.Path:
     if not object_typename or not object_path:
@@ -468,6 +472,7 @@ def get_json_data_file(object_json: pathlib.Path,
 
     return None
 
+
 def get_json_data(object_typename: str,
                   object_path: str or pathlib.Path,
                   object_validator: callable) -> dict or None:
@@ -538,6 +543,7 @@ def get_restricted_json_data(restricted_name: str = None, restricted_path: str o
 
     return get_json_data('restricted', restricted_path, validation.valid_o3de_restricted_json)
 
+
 def get_repo_json_data(repo_uri: str) -> dict or None:
     if not repo_uri:
         logger.error('Must specify a Repo Uri.')
@@ -547,7 +553,8 @@ def get_repo_json_data(repo_uri: str) -> dict or None:
 
     return get_json_data_file(repo_json, "Repo", validation.valid_o3de_repo_json)
 
-def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
+
+def get_repo_path(repo_uri: str, cache_folder: str or pathlib.Path = None) -> pathlib.Path:
     if not cache_folder:
         cache_folder = get_o3de_cache_folder()
 
@@ -555,6 +562,7 @@ def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
     repo_sha256 = hashlib.sha256(repo_manifest.encode())
     return cache_folder / str(repo_sha256.hexdigest() + '.json')
 
+
 def get_registered(engine_name: str = None,
                    project_name: str = None,
                    gem_name: str = None,

scripts/o3de/o3de/print_registration.py

@@ -14,10 +14,10 @@ import pathlib
 import sys
 import urllib.parse
 
-from o3de import manifest, validation
+from o3de import manifest, validation, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.print_registration')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def get_project_path(project_path: pathlib.Path, project_name: str) -> pathlib.Path:

scripts/o3de/o3de/project_properties.py

@@ -15,8 +15,9 @@ import logging
 
 from o3de import manifest, utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.project_properties')
+logging.basicConfig(format=utils.LOG_FORMAT)
+
 
 def get_project_props(name: str = None, path: pathlib.Path = None) -> dict:
     proj_json = manifest.get_project_json_data(project_name=name, project_path=path)
@@ -26,6 +27,7 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict:
         return None
     return proj_json
 
+
 def edit_project_props(proj_path: pathlib.Path = None,
                        proj_name: str = None,
                        new_name: str = None,
@@ -71,9 +73,9 @@ def edit_project_props(proj_path: pathlib.Path = None,
         tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags
         proj_json['user_tags'] = tag_list
 
-
     return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1
 
+
 def _edit_project_props(args: argparse) -> int:
     return edit_project_props(args.project_path,
                               args.project_name,
@@ -86,6 +88,7 @@ def _edit_project_props(args: argparse) -> int:
                               args.delete_tags,
                               args.replace_tags)
 
+
 def add_parser_args(parser):
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
@@ -112,10 +115,12 @@ def add_parser_args(parser):
                        help='Replace entirety of user_tags property with space delimited list of values')
     parser.set_defaults(func=_edit_project_props)
 
+
 def add_args(subparsers) -> None:
     enable_project_props_subparser = subparsers.add_parser('edit-project-properties')
     add_parser_args(enable_project_props_subparser)
-    
+
+
 def main():
     the_parser = argparse.ArgumentParser()
     add_parser_args(the_parser)
@@ -123,5 +128,6 @@ def main():
     ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
     sys.exit(ret)
 
+
 if __name__ == "__main__":
     main()

scripts/o3de/o3de/register.py

@@ -23,8 +23,8 @@ import urllib.request
 
 from o3de import get_registration, manifest, repo, utils, validation
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.register')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def register_shipped_engine_o3de_objects(force: bool = False) -> int:
@@ -306,9 +306,9 @@ def register_o3de_object_path(json_data: dict,
         try:
             paths_to_remove.append(o3de_object_path.relative_to(save_path.parent))
         except ValueError:
-            pass # It is not an error if a relative path cannot be formed
+            pass  # It is not an error if a relative path cannot be formed
     manifest_data[o3de_object_key] = list(filter(lambda p: pathlib.Path(p) not in paths_to_remove,
-                                                           manifest_data.setdefault(o3de_object_key, [])))
+                                                 manifest_data.setdefault(o3de_object_key, [])))
 
     if remove:
         if save_path:
@@ -732,14 +732,14 @@ def register(engine_path: pathlib.Path = None,
             logger.error(f'Gem path cannot be empty.')
             return 1
         result = result or register_gem_path(json_data, gem_path, remove,
-                                   external_subdir_engine_path, external_subdir_project_path)
+                                             external_subdir_engine_path, external_subdir_project_path)
 
     if isinstance(external_subdir_path, pathlib.PurePath):
         if not external_subdir_path:
             logger.error(f'External Subdirectory path is None.')
             return 1
         result = result or register_external_subdirectory(json_data, external_subdir_path, remove,
-                                                external_subdir_engine_path, external_subdir_project_path)
+                                                          external_subdir_engine_path, external_subdir_project_path)
 
     if isinstance(template_path, pathlib.PurePath):
         if not template_path:
@@ -841,6 +841,10 @@ def add_parser_args(parser):
     Ex. Directly run from this file alone with: python register.py --engine-path "C:/o3de"
     :param parser: the caller passes an argparse parser like instance to this method
     """
+
+    # Sub-commands should declare their own verbosity flag, if desired
+    utils.add_verbosity_arg(parser)
+
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('--this-engine', action='store_true', required=False,
                        default=False,
@@ -888,18 +892,18 @@ def add_parser_args(parser):
                        help='Refresh the repo cache.')
 
     parser.add_argument('-ohf', '--override-home-folder', type=pathlib.Path, required=False,
-                                    help='By default the home folder is the user folder, override it to this folder.')
+                        help='By default the home folder is the user folder, override it to this folder.')
     parser.add_argument('-r', '--remove', action='store_true', required=False,
-                                    default=False,
-                                    help='Remove entry.')
+                        default=False,
+                        help='Remove entry.')
     parser.add_argument('-f', '--force', action='store_true', default=False,
-                                    help='For the update of the registration field being modified.')
+                        help='For the update of the registration field being modified.')
 
-    external_subdir_group =  parser.add_argument_group(title='external-subdirectory',
-                                                       description='path arguments to use with the --external-subdirectory option')
+    external_subdir_group = parser.add_argument_group(title='external-subdirectory',
+                                                      description='path arguments to use with the --external-subdirectory option')
     external_subdir_path_group = external_subdir_group.add_mutually_exclusive_group()
     external_subdir_path_group.add_argument('-esep', '--external-subdirectory-engine-path', type=pathlib.Path,
-                                       help='If supplied, registers the external subdirectory with the engine.json at' \
+                                            help='If supplied, registers the external subdirectory with the engine.json at' \
                                             ' the engine-path location')
     external_subdir_path_group.add_argument('-espp', '--external-subdirectory-project-path', type=pathlib.Path)
     parser.set_defaults(func=_run_register)
@@ -924,8 +928,6 @@ def main():
     # parse the command line args
     the_parser = argparse.ArgumentParser()
 
-    # add subparsers
-
     # add args to the parser
     add_parser_args(the_parser)
 
@@ -934,6 +936,7 @@ def main():
 
     # run
     ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
+    logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
 
     # return
     sys.exit(ret)

scripts/o3de/o3de/repo.py

@@ -15,8 +15,8 @@ import hashlib
 from datetime import datetime
 from o3de import manifest, utils, validation
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.repo')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def process_add_o3de_repo(file_name: str or pathlib.Path,

scripts/o3de/o3de/sha256.py

@@ -15,8 +15,8 @@ import sys
 
 from o3de import utils
 
-logger = logging.getLogger()
-logging.basicConfig()
+logger = logging.getLogger('o3de.sha256')
+logging.basicConfig(format=utils.LOG_FORMAT)
 
 
 def sha256(file_path: str or pathlib.Path,
@@ -35,7 +35,7 @@ def sha256(file_path: str or pathlib.Path,
             logger.error(f'Json path {json_path} does not exist.')
             return 1
 
-    sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest()
+    the_sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest()
 
     if json_path:
         with json_path.open('r') as s:
@@ -44,7 +44,7 @@ def sha256(file_path: str or pathlib.Path,
             except json.JSONDecodeError as e:
                 logger.error(f'Failed to read Json path {json_path}: {str(e)}')
                 return 1
-        json_data.update({"sha256": sha256})
+        json_data.update({"sha256": the_sha256})
         utils.backup_file(json_path)
         with json_path.open('w') as s:
             try:
@@ -53,7 +53,7 @@ def sha256(file_path: str or pathlib.Path,
                 logger.error(f'Failed to write Json path {json_path}: {str(e)}')
                 return 1
     else:
-        print(sha256)
+        print(the_sha256)
     return 0
 
 
@@ -70,9 +70,9 @@ def add_parser_args(parser):
     :param parser: the caller passes an argparse parser like instance to this method
     """
     parser.add_argument('-f', '--file-path', type=str, required=True,
-                                  help='The path to the file you want to sha256.')
+                        help='The path to the file you want to sha256.')
     parser.add_argument('-j', '--json-path', type=str, required=False,
-                                  help='optional path to an o3de json file to add the "sha256" element to.')
+                        help='Optional path to an o3de json file to add the "sha256" element to.')
     parser.set_defaults(func=_run_sha256)
 
 

scripts/o3de/o3de/utils.py

@@ -8,6 +8,7 @@
 """
 This file contains utility functions
 """
+import argparse
 import sys
 import uuid
 import os
@@ -17,11 +18,54 @@ import urllib.request
 import logging
 import zipfile
 
-logger = logging.getLogger()
-logging.basicConfig()
+LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s'
+
+logger = logging.getLogger('o3de.utils')
+logging.basicConfig(format=LOG_FORMAT)
 
 COPY_BUFSIZE = 64 * 1024
 
+
+class VerbosityAction(argparse.Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super().__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help,
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        count = getattr(namespace, self.dest, None)
+        if count is None:
+            count = self.default
+        count += 1
+        setattr(namespace, self.dest, count)
+        # get the parent logger instance
+        log = logging.getLogger('o3de')
+        if count >= 2:
+            log.setLevel(logging.DEBUG)
+        elif count == 1:
+            log.setLevel(logging.INFO)
+
+
+def add_verbosity_arg(parser: argparse.ArgumentParser) -> None:
+    """
+    Add a consistent/common verbosity option to an arg parser
+    :param parser: The ArgumentParser to modify
+    :return: None
+    """
+    parser.add_argument('-v', dest='verbosity', action=VerbosityAction, default=0,
+                        help='Additional logging verbosity, can be -v or -vv')
+
+
 def copyfileobj(fsrc, fdst, callback, length=0):
     # This is functionally the same as the python shutil copyfileobj but
     # allows for a callback to return the download progress in blocks and allows
@@ -43,10 +87,11 @@ def copyfileobj(fsrc, fdst, callback, length=0):
             return 1
     return 0
 
+
 def validate_identifier(identifier: str) -> bool:
     """
-    Determine if the identifier supplied is valid.
-    :param identifier: the name which needs to to checked
+    Determine if the identifier supplied is valid
+    :param identifier: the name which needs to be checked
     :return: bool: if the identifier is valid or not
     """
     if not identifier:
@@ -65,7 +110,7 @@ def validate_identifier(identifier: str) -> bool:
 def sanitize_identifier_for_cpp(identifier: str) -> str:
     """
     Convert the provided identifier to a valid C++ identifier
-    :param identifier: the name which needs to to sanitized
+    :param identifier: the name which needs to be sanitized
     :return: str: sanitized identifier
     """
     if not identifier:
@@ -81,8 +126,8 @@ def sanitize_identifier_for_cpp(identifier: str) -> str:
 
 def validate_uuid4(uuid_string: str) -> bool:
     """
-    Determine if the uuid supplied is valid.
-    :param uuid_string: the uuid which needs to to checked
+    Determine if the uuid supplied is valid
+    :param uuid_string: the uuid which needs to be checked
     :return: bool: if the uuid is valid or not
     """
     try:
@@ -117,11 +162,13 @@ def backup_folder(folder: str or pathlib.Path) -> None:
             if backup_folder_name.is_dir():
                 renamed = True
 
+
 def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool = False, download_progress_callback = None) -> int:
     """
+    Download file
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_path: location path on disk to download file
-    :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
+    :param download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
     """
     if download_path.is_file():
         if not force_overwrite:
@@ -142,10 +189,12 @@ def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool
                     download_file_size = s.headers['content-length']
                 except KeyError:
                     pass
+
                 def download_progress(downloaded_bytes):
                     if download_progress_callback:
                         return download_progress_callback(int(downloaded_bytes), int(download_file_size))
                     return False
+
                 with download_path.open('wb') as f:
                     download_cancelled = copyfileobj(s, f, download_progress)
                     if download_cancelled:
@@ -183,13 +232,12 @@ def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, force_overwri
 def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path,
                        max_scan_up_range: int=0) -> pathlib.Path or None:
     """
-    Find a file with the given name in the ancestor directories by walking up the starting path until the file is found.
-
-    :param target_file_name: Name of the file to find.
-    :param start_path: path to start looking for the file.
+    Find a file with the given name in the ancestor directories by walking up the starting path until the file is found
+    :param target_file_name: Name of the file to find
+    :param start_path: path to start looking for the file
     :param max_scan_up_range: maximum number of directories to scan upwards when searching for target file
            if the value is 0, then there is no max
-    :return: Path to the file or None if not found.
+    :return: Path to the file or None if not found
     """
     current_path = pathlib.Path(start_path)
     candidate_path = current_path / target_file_name
@@ -211,17 +259,17 @@ def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.P
 
     return candidate_path if candidate_path.exists() else None
 
+
 def find_ancestor_dir_containing_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path,
                                       max_scan_up_range: int=0) -> pathlib.Path or None:
     """
-    Find nearest ancestor directory that contains the file with the given name by walking up
-    from the starting path.
-
-    :param target_file_name: Name of the file to find.
-    :param start_path: path to start looking for the file.
+    Find the nearest ancestor directory that contains the file with the given name by walking up
+    from the starting path
+    :param target_file_name: Name of the file to find
+    :param start_path: path to start looking for the file
     :param max_scan_up_range: maximum number of directories to scan upwards when searching for target file
            if the value is 0, then there is no max
-    :return: Path to the directory containing file or None if not found.
+    :return: Path to the directory containing file or None if not found
     """
     ancestor_file = find_ancestor_file(target_file_name, start_path, max_scan_up_range)
     return ancestor_file.parent if ancestor_file else None

scripts/o3de/o3de/validation.py

@@ -11,6 +11,7 @@ This file validating o3de object json files
 import json
 import pathlib
 
+
 def valid_o3de_json_dict(json_data: dict, key: str) -> bool:
     return key in json_data
 
@@ -23,9 +24,9 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool:
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['repo_name']
-            test = json_data['origin']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['repo_name']
+            _ = json_data['origin']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True
 
@@ -38,8 +39,8 @@ def valid_o3de_engine_json(file_name: str or pathlib.Path) -> bool:
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['engine_name']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['engine_name']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True
 
@@ -52,8 +53,8 @@ def valid_o3de_project_json(file_name: str or pathlib.Path) -> bool:
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['project_name']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['project_name']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True
 
@@ -66,8 +67,8 @@ def valid_o3de_gem_json(file_name: str or pathlib.Path) -> bool:
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['gem_name']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['gem_name']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True
 
@@ -76,11 +77,12 @@ def valid_o3de_template_json(file_name: str or pathlib.Path) -> bool:
     file_name = pathlib.Path(file_name).resolve()
     if not file_name.is_file():
         return False
+
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['template_name']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['template_name']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True
 
@@ -89,10 +91,11 @@ def valid_o3de_restricted_json(file_name: str or pathlib.Path) -> bool:
     file_name = pathlib.Path(file_name).resolve()
     if not file_name.is_file():
         return False
+
     with file_name.open('r') as f:
         try:
             json_data = json.load(f)
-            test = json_data['restricted_name']
-        except (json.JSONDecodeError, KeyError) as e:
+            _ = json_data['restricted_name']
+        except (json.JSONDecodeError, KeyError):
             return False
     return True