Make scrubber/validator not depend on legacy packaging scripts (#6053)

* Make validator not depend on legacy packaging scripts

Signed-off-by: Shirang Jia <[email protected]>

* Remove unused glob_to_regex.py

Signed-off-by: Shirang Jia <[email protected]>

* Remove unsued import path

Signed-off-by: Shirang Jia <[email protected]>

scripts/build/package/glob_to_regex.py

@@ -1,130 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-from __future__ import absolute_import
-import os
-import re
-import json
-import sys
-try:
-    import six
-except ImportError:
-    import pip
-    pip.main(['install', 'six', '--ignore-installed', '-q'])
-    import six
-from pathlib import Path
-
-this_file_path = os.path.dirname(os.path.realpath(__file__))
-
-# resolve symlinks and eliminate ".." components
-engine_root_path = Path(__file__).resolve().parents[3]
-    
-def convert_glob_pattern_to_regex_pattern(glob_pattern):
-    # switch to forward slashes because way easier to pattern match against
-    pattern = re.sub(r'\\', r'/', glob_pattern)
-
-    # Replace the dots and question marks
-    pattern = re.sub(r'\.', r'\\.', pattern)
-    pattern = re.sub(r'\?', r'.', pattern)
-
-    # Handle the * vs ** expansions
-    pattern = re.sub(r'([^*])\*($|[^*])', r'\1[^/\\\\]*\2', pattern)
-    pattern = re.sub(r'\*\*/', r'(.*/)?', pattern)
-    pattern = re.sub(r'\*\*', r'.*', pattern)
-
-    # replace the forward slashes with [/\\] so it works on PC/unix
-    pattern = re.sub(r'([^^])/', r'\1[/\\\\]', pattern)
-    return pattern
-
-# Convert the package json into a pair of regexes we can use to look for includes and excludes
-def convert_glob_list_to_regex_list(filelist, prefix):
-    includes = []
-    excludes = []
-    for key, value in six.iteritems(filelist):
-        glob_pattern = os.path.join(prefix, key)
-        if isinstance(value, dict):
-            (sub_includes, sub_excludes) = convert_glob_list_to_regex_list(value, glob_pattern)
-            includes.extend(sub_includes)
-            excludes.extend(sub_excludes)
-        else:
-            # Simulate what glob would do with file walking to scope the * within a directory
-            # and ** across directories
-            regex_pattern = convert_glob_pattern_to_regex_pattern(os.path.normpath(glob_pattern))
-
-            # Deal with the commands. include/exclude are straight forward. Moves/renames are to be considered
-            # includes, and we will stick with validating the original contents for now
-            if value == "#include":
-                includes.append(regex_pattern)
-            elif value == "#exclude":
-                excludes.append(regex_pattern)
-            elif value.startswith('#move:'):
-                includes.append(regex_pattern)
-            elif value.startswith('#rename:'):
-                includes.append(regex_pattern)
-            else:
-                pass
-    return (includes, excludes)
-
-def generate_excludes_for_platform(root, platform):
-    if platform == 'all':
-        platform_exclusions_filename = os.path.join(this_file_path, 'platform_exclusions.json')
-        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
-            platform_exclusions = json.load(platform_exclusions_file)
-    else:
-        # Use real path in case root is a symlink path
-        if os.name == 'posix' and os.path.islink(root):
-            root = os.readlink(root)
-        # "root" is the root of the folder structure we're validating 
-        # "engine_root_path" is the engine root where the restricted platform folder is linked
-        relative_folder = os.path.relpath(this_file_path, engine_root_path)
-        platform_exclusions_filename = os.path.join(engine_root_path, 'restricted', platform, relative_folder, platform.lower() + '_exclusions.json')
-        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
-            platform_exclusions = json.load(platform_exclusions_file)
-        
-    if platform not in platform_exclusions:
-        raise KeyError('No {} found in {}'.format(platform, platform_exclusions_filename))
-    if '@lyengine' not in platform_exclusions[platform]:
-        raise KeyError('No {}/@lyengine found in {}'.format(platform, package_file_list))
-    (_, excludes) = convert_glob_list_to_regex_list(platform_exclusions[platform]['@lyengine'], root)
-    del _
-    return excludes
-
-def generate_include_exclude_regexes(package_platform, package_type, root, prohibited_platforms):
-    # The general contents will be indicated by the package file
-    if package_type == 'all':
-        package_file_list = os.path.join(this_file_path, 'package_filelists', 'all.json')
-    else:
-        # Search non-restricted platform first
-        package_file_list = os.path.join(this_file_path, 'Platform', package_platform, 'package_filelists', f'{package_type}.json')
-        if not os.path.exists(filelist):
-            # Use real path in case root is a symlink path
-            if os.name == 'posix' and os.path.islink(root):
-                root = os.readlink(root)
-            # "root" is the root of the folder structure we're validating
-            # "engine_root_path" is the engine root where the restricted platform folder is linked
-            rel_path = os.path.relpath(this_file_path, engine_root_path)
-            package_file_list = os.path.join(engine_root_path, 'restricted', package_platform, rel_path, 'package_filelists',
-                                    f'{package_type}.json')
-    with open(package_file_list, 'r') as package_file:
-        package = json.load(package_file)
-
-    if '@lyengine' not in package:
-        raise KeyError('No @lyengine found in {}'.format(package_file_list))
-
-    (includes_list, excludes_list) = convert_glob_list_to_regex_list(package['@lyengine'], root)
-    prohibited_platforms.append('all')
-
-    # Add the exclusions of each prohibited platform
-    for p in prohibited_platforms:
-        excludes_list.extend(generate_excludes_for_platform(root, p))
-
-    includes = re.compile('|'.join(includes_list), re.IGNORECASE)
-    excludes = re.compile('|'.join(excludes_list), re.IGNORECASE)
-    return (includes, excludes)
-
-def generate_exclude_regexes_for_platform(root, platform):
-    return re.compile('|'.join(generate_excludes_for_platform(root, platform)), re.IGNORECASE)

scripts/scrubbing/scrubbing_job.py

@@ -1,29 +0,0 @@
-#
-# Copyright (c) Contributors to the Open 3D Engine Project.
-# For complete copyright and license terms please see the LICENSE at the root of this distribution.
-#
-# SPDX-License-Identifier: Apache-2.0 OR MIT
-#
-#
-
-import os
-import sys
-cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.abspath(f'{cur_dir}/../build/package'))
-import util
-
-# Run validator
-success = True
-validator_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'validator.py')
-engine_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-if sys.platform == 'win32':
-    python = os.path.join(engine_root, 'python', 'python.cmd')
-else:
-    python = os.path.join(engine_root, 'python', 'python.sh')
-args = [python, validator_path, '--package_platform', 'Windows', '--package_type', 'all', engine_root]
-return_code = util.safe_execute_system_call(args)
-if return_code != 0:
-    success = False
-if not success:
-    util.error('Restricted file validator failed.')
-print('Restricted file validator completed successfully.')

scripts/scrubbing/validator.py

@@ -29,8 +29,6 @@ else:
     from io import StringIO
 
 import validator_data_LEGAL_REVIEW_REQUIRED # pull in the data we need to configure this tool
-sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'build', 'package'))
-from glob_to_regex import generate_include_exclude_regexes
 
 class Validator(object):
     """Class to contain the validator program"""
@@ -212,9 +210,6 @@ class Validator(object):
     # TODO: Perhaps the directories to skip should become a parameter so we can use the validator
     # on non-Lumberyard trees.
     def validate_directory_tree(self, root, platform):
-        prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(self.options.package_platform)
-        (includes, excludes) = generate_include_exclude_regexes(self.options.package_platform, self.options.package_type, root, prohibited_platforms)
-
         """Walk from root to find all files to validate and call the validator on each file.
         Return 0 if no problems where found, and 1 if any validation failures occured."""
         counter = 0
@@ -227,28 +222,22 @@ class Validator(object):
             # First deal with the files in the current directory
             for filename in filenames:
                 filepath = os.path.join(dirname, filename)
-                include_match = includes.match(filepath)
-                exclude_match = excludes.match(filepath)
-                allowed = include_match and not exclude_match
-
-                if self.options.all or allowed:
-                    scanned += 1
-                    file_failed = self.validate_file(os.path.normpath(filepath))
-                    if file_failed:
-                        platform_failed = file_failed
-                    else:
-                        validations += 1
-                counter += 1
+                scanned += 1
+                file_failed = self.validate_file(os.path.normpath(filepath))
+                if file_failed:
+                    platform_failed = file_failed
+                else:
+                    validations += 1
 
             # Trim out allowlisted subdirectories in the current directory if allowed
             for name in bypassed_directories:
                 if name in dirnames:
                     dirnames.remove(name)
-        if counter == 0 or scanned == 0:
+        if scanned == 0:
             logging.error('No files scanned at target search directory: %s', root)
             platform_failed = 1
         else:
-            print('validated {} of {} package files ({} non-package files skipped)'.format(validations, scanned, counter - scanned))
+            print('validated {} of {} files'.format(validations, scanned))
         return platform_failed
 
 
@@ -387,8 +376,6 @@ def parse_options():
                       choices=platform_choices,
                       dest='package_platform',
                       help='Package platform to validate. Must be one of {}.'.format(platform_choices))
-    parser.add_option('--package_type', action='store', type='string', default='all', dest='package_type',
-                      help='Package type to validate.')
     parser.add_option('-s', '--store-exceptions', action='store', type='string', default='',
                       dest='exception_file',
                       help='Store list of lines that the validator gave exceptions to by matching accepted use patterns. These can be diffed with prior runs to see what is changing.')
@@ -430,7 +417,6 @@ def main():
 
     package_failed = 0
     package_platform = validator.options.package_platform
-    package_type = validator.options.package_type
     prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(package_platform)
 
     if validator.options.exception_file != '':
@@ -441,19 +427,20 @@ def main():
             sys.exit(1)
 
     for platform in prohibited_platforms:
-        print('validating {} against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
+        print('validating {} against {} for package platform {}'.format(args[0], platform, package_platform))
         platform_failed = validator.validate(platform)
         if platform_failed:
-            print('{} FAILED validation against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
+            print('{} FAILED validation against {} for package platform {}'.format(args[0], platform, package_platform))
             package_failed = platform_failed
         else:
-            print('{} is VALIDATED against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
+            print('{} is VALIDATED against {} for package platform {}'.format(args[0], platform, package_platform))
 
     if validator.options.exception_file != '':
         validator.exceptions_output.close()
 
     return package_failed
 
+
 if __name__ == '__main__':
     # pylint: disable-msg=C0103
     main_results = main()