123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171 |
- #
- # Copyright (c) Contributors to the Open 3D Engine Project.
- # For complete copyright and license terms please see the LICENSE at the root of this distribution.
- #
- # SPDX-License-Identifier: Apache-2.0 OR MIT
- #
- #
- import argparse
- import hashlib
- import os
- import pathlib
- import platform
- import urllib.request
- import subprocess
- import sys
- import zipfile
- SUPPORTED_HASH_ALGORITHMS = {
- 'md5': lambda: hashlib.md5(),
- 'sha1': lambda: hashlib.sha1(),
- 'sha224': lambda: hashlib.sha224(),
- 'sha256': lambda: hashlib.sha256()
- }
- ARCHIVE_EXTS_ZIP = { '.zip' }
- ARCHIVE_EXTS_TAR = { '.tgz', 'gz', '.xz', '.tar.xz' }
- ARCHIVE_EXTS_7ZIP = { '.7z' }
- INDENT=' '*4
- def hash_file(file_path:str, hash_algorithm:str='md5')->str:
- """
- Calculate a hash based on the input file path and selected hash algorithm and return a hex-string representation of it
- (Refer to SUPPORTED_HASH_ALGORITHMS for the supported hash algorithms)
- :param file_path: The path to the file to calculate the hash
- :param hash_algorithm: The desired hash algorith. See 'SUPPORTED_HASH_ALGORITHMS' for the list of algorithms
- """
- if not os.path.isfile(file_path):
- raise FileNotFoundError(f"File to hash {file_path} does not exist.")
- hasher_create = SUPPORTED_HASH_ALGORITHMS.get(hash_algorithm)
- if not hasher_create:
- raise KeyError("Invalid hash algorithm selected for hash calculation: '{hash_algorithm}'")
- hasher = hasher_create()
-
- # we don't follow symlinks here, this is strictly to check actual packages.
- with open(file_path, 'rb') as afile:
- buf = afile.read()
- hasher.update(buf)
- hash_result = hasher.hexdigest()
- return hash_result
- def download_and_verify(src_url: str, src_zip_hash:str, src_zip_hash_algorithm:str,target_folder:str)->str:
- """
- Calculate a hash based on the input file path and selected hash algorithm and return a hex-string representation of it
- Supported hash algorithms are: md5, sha1, sha224, sha256
- :param src_url: The full url of the archive file to download
- :param src_zip_hash: The expected hash to use to verify the integrity of the downloaded file. If 'None', then
- verification is replaced by the calculation and reporting of the result hash of the downloaded package.
- :param src_zip_hash_algorithm: The desired hash algorith. See 'SUPPORTED_HASH_ALGORITHMS' for the list of algorithms
- :param target_folder: The target folder to download the archive file to
- """
- target_folder_path = pathlib.Path(target_folder)
- src_filename = os.path.basename(src_url)
- tgt_filename = target_folder_path / src_filename
- # If the file has been downloaded, check its hash
- current_hash = None
- if tgt_filename.is_file():
- current_hash = hash_file(file_path=str(tgt_filename),
- hash_algorithm=src_zip_hash_algorithm)
- print(f"Current hash of {tgt_filename}:{current_hash}")
- if current_hash and current_hash == src_zip_hash:
- print(f"{INDENT}File '{src_filename}' already downloaded to {tgt_filename}, skipping.")
- return str(tgt_filename)
- print(f"{INDENT}Downloading {src_url}")
- if tgt_filename.exists():
- tgt_filename.unlink()
- urllib.request.urlretrieve(src_url, tgt_filename)
- # Calculate the downloaded file hash
- downloaded_hash = hash_file(file_path=str(tgt_filename),
- hash_algorithm=src_zip_hash_algorithm)
- if src_zip_hash and src_zip_hash != downloaded_hash:
- raise RuntimeError(f"Hash {src_zip_hash_algorithm} verification failed for {tgt_filename} (downloaded hash: {downloaded_hash}")
- print(f"{INDENT}Package hash : ({src_zip_hash_algorithm}) {downloaded_hash}")
- return str(tgt_filename)
- def extract_package(src_package_file: str, target_folder:str):
- src_package_file_path = pathlib.Path(src_package_file)
- target_folder_path = pathlib.Path(target_folder)
- if not src_package_file_path.is_file():
- raise FileNotFoundError(f"Package to extract '{src_package_file_path}' does not exist.")
-
- package_name, package_ext = os.path.splitext(str(src_package_file_path.name))
- destination_path = target_folder_path / package_name
- print(f"{INDENT}Extracting {src_package_file_path} to {destination_path}")
- if package_ext in ARCHIVE_EXTS_ZIP:
- import zipfile
- with zipfile.ZipFile(str(src_package_file_path.resolve()), 'r') as dep_zip:
- dep_zip.extractall(destination_path)
- elif package_ext in ARCHIVE_EXTS_TAR:
- import tarfile
- with tarfile.open(str(src_package_file_path.resolve())) as tar_file:
- tarfile.extractall(destination_path)
- elif package_ext in ARCHIVE_EXTS_7ZIP:
- try:
- os.makedirs(destination_path, exist_ok=True)
- subprocess.call(['7z', 'x', '-y', str(src_package_file_path.resolve())], cwd=destination_path)
- except Exception:
- raise RuntimeError(f"Archive file {src_package_file_path} requires 7Zip to be installed and on the command path. ")
- else:
- print(f"Extracted to {destination_path}")
-
- else:
- raise RuntimeError(f"Unsupported package extension: {package_ext}")
- return destination_path
- if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Download, verify hash, and unpack remote zip file")
- parser.add_argument('src-url',
- help='The download url of the zip package to download',
- nargs=1)
- parser.add_argument('--hash-algorithm',
- help=f'The hash algorithm to use to calculate the fingerprint ({" ".join(SUPPORTED_HASH_ALGORITHMS.keys())})',
- default='sha256',
- required=False)
- parser.add_argument('--hash',
- help='The hash fingerprint to validate against',
- default='',
- required=False)
- parser.add_argument('--target-folder',
- help='The target location for the download',
- required=True)
-
- parsed_args = parser.parse_args()
- downloaded_package_file = download_and_verify(src_url=parsed_args.src_url[0],
- src_zip_hash=parsed_args.hash,
- src_zip_hash_algorithm=parsed_args.hash_algorithm,
- target_folder=parsed_args.target_folder)
- extracted_package_path = extract_package(src_package_file=downloaded_package_file,
- target_folder=parsed_args.target_folder)
- sys.exit(1)
|