#!/usr/bin/env python3 # # Copyright (c) Contributors to the Open 3D Engine Project. # For complete copyright and license terms please see the LICENSE at the root of this distribution. # # SPDX-License-Identifier: Apache-2.0 OR MIT # # import argparse import logging import os import pathlib import platform import requests import shutil import string import subprocess import tarfile import time SCRIPT_PATH = pathlib.Path(__file__).parent PATCH_FILE = SCRIPT_PATH / "mcpp_2.7.2_az.patch" SOURCE_NAME = "mcpp-2.7.2" SOURCE_TAR_FILE = f"{SOURCE_NAME}.tar.gz" SOURCEFORGE_URL = "https://sourceforge.net/projects/mcpp/files/mcpp/V.2.7.2" SOURCEFORGE_DOWNLOAD_URL = f"{SOURCEFORGE_URL}/{SOURCE_TAR_FILE}/download" PLATFORM_LINUX = 'linux' PLATFORM_LINUX_ARM64 = 'linux-aarch64' PLATFORM_MAC = 'mac' PLATFORM_WINDOWS = 'windows' if platform.system() == 'Linux': if platform.processor() == 'aarch64': platform_name = PLATFORM_LINUX_ARM64 else: platform_name = PLATFORM_LINUX shared_lib_name = 'libmcpp.so' static_lib_name = 'libmcpp.a' elif platform.system() == 'Darwin': platform_name = PLATFORM_MAC shared_lib_name = 'libmcpp.dylib' static_lib_name = 'libmcpp.a' elif platform.system() == 'Windows': platform_name = PLATFORM_WINDOWS static_lib_name = 'mcpp0.lib' shared_lib_name = 'mcpp0.dll' # Note: If we are running from windows, this script must be run from a visual studio command prompt # We will check the system environment to make sure that 'INCLUDE', 'LIB', 'LIBPATH' are set for vs_env_key in ('INCLUDE', 'LIB', 'LIBPATH'): if os.environ.get(vs_env_key) is None: print("This script must be run from a visual studio command prompt, or the visual studio command line" " environments must be set") exit(1) # Check it's running under x64 build environment. vs_target_arch = os.environ.get('VSCMD_ARG_TGT_ARCH') if vs_target_arch is None: print("Couldn't read the environment variable 'VSCMD_ARG_TGT_ARCH'. This script must be run from a x64 visual studio command prompt, or the visual studio command line" " environments must be set") exit(1) if vs_target_arch != 'x64': print("This script must be run from a x64 visual studio command prompt, or the visual studio command line" " environments must be set") exit(1) else: assert False, "Invalid platform" assert platform_name in (PLATFORM_LINUX, PLATFORM_LINUX_ARM64, PLATFORM_MAC, PLATFORM_WINDOWS), f"Invalid platform_name {platform_name}" TARGET_3PP_PACKAGE_FOLDER = SCRIPT_PATH.parent / f'mcpp-{platform_name}' MCPP_DETAIL = f""" The MCPP package will be patched and built from the following sources Sourceforge URL : {SOURCEFORGE_URL} Source name : {SOURCE_NAME} Patch File : {PATCH_FILE} Target Pre-package Platform target : {TARGET_3PP_PACKAGE_FOLDER} Example command: python get_and_build_linux.sh mcpp-2.7.2_az.1-rev1-{platform_name} """ def execute_cmd(cmd, cwd=None, shell=False, suppress_std_err=False, env=None): logging.debug(f"[DEBUG] Calling {subprocess.list2cmdline(cmd)}") if shell: cmd = subprocess.list2cmdline(cmd) return subprocess.call(cmd, shell=shell, cwd=cwd, env=env, stderr=subprocess.DEVNULL if suppress_std_err else subprocess.STDOUT) def prepare_temp_folder(temp_folder): if temp_folder.exists(): shutil.rmtree(str(temp_folder.resolve()), ignore_errors=True) os.makedirs(temp_folder, exist_ok=True) return True def download_from_source_forge(source_forge_download_url, temp_folder): try: request = requests.get(source_forge_download_url, allow_redirects=True) target_file = temp_folder / SOURCE_TAR_FILE if target_file.is_file(): target_file.unlink() with open(str(target_file.resolve()), 'wb') as request_stream: bytes_written = request_stream.write(request.content) logging.debug(f'[DEBUG] {SOURCE_TAR_FILE} downloaded ({bytes_written} bytes)') return True except Exception as e: logging.fatal(f'[FATAL] Error downloading from {source_forge_download_url} : {e}') return False def extract_tarfile(temp_folder): try: target_file = temp_folder / SOURCE_TAR_FILE if not target_file.is_file(): logging.error(f'[ERROR] Missing expected tar file {target_file}') tar = tarfile.open(str(target_file), 'r:gz') tar.extractall(path=str(temp_folder.resolve())) logging.debug(f'[DEBUG] {SOURCE_TAR_FILE} extracted to ({str(temp_folder.resolve())})') return True except Exception as e: logging.fatal(f'[FATAL] extracting tar file {target_file} : {e}') return False def init_git_repo(temp_folder): """ Runs 'git init' on temp_folder. This is useful to 'git apply' won't fail silently when executing apply_patch(...) REMARK: 1- It is very important to set the local git project to not autoconvert LF to CRLF because it causes the patching to fail as 'git apply' is very picky about that. 2- You may notice that there's a .gitattributes file that makes sure the patch file remains with LF when fetched from the repo. 3- It was also found that 'git apply' also failed if the patch had CRLF AND the local git repo also had CRLF. """ pristine_source_path = str((temp_folder / SOURCE_NAME).resolve()) git_cmds = [ ['git', 'init'], ['git', 'config', '--local', 'core.eol', 'lf'], ['git', 'config', '--local', 'core.autocrlf', 'false'], ['git', 'add', '.'], ['git', 'commit', '--no-verify', '-m', 'Temporary Message'], ] try: # Check for git --version to make sure it's installed. result = execute_cmd(['git', '--version'], shell=True, suppress_std_err=True) if result != 0: raise Exception("'git' command was not found") for git_cmd in git_cmds: result = execute_cmd(git_cmd, shell=True, cwd=pristine_source_path) if result != 0: cmd_string = " ".join(git_cmd) raise Exception(f"The command '{cmd_string}' failed to execute") return True except Exception as e: logging.fatal(f'[FATAL] Error initializing git repo : {e}') return False def apply_patch(temp_folder, patch_file): pristine_source_path = str((temp_folder / SOURCE_NAME).resolve()) # Git apply for some reason fails on windows, but works on other platforms. We will first try 'git', and if that # fails, we will try 'patch' apply_patch_cmds = [ ('git', ['apply', str(patch_file.resolve())]), ('patch', ['--strip=1', f'--input={str(patch_file.resolve())}']) ] try: patched = False for apply_patch_cmd in apply_patch_cmds: patch_cmd = apply_patch_cmd[0] logging.info(f"Attempt to patch with {patch_cmd}") result = execute_cmd([patch_cmd, '--version'], shell=True, suppress_std_err=True) if result != 0: logging.debug(f"[DEBUG] Unable to locate cmd {patch_cmd} for patching.") continue patch_full_cmd = [patch_cmd] patch_full_cmd.extend(apply_patch_cmd[1]) result = execute_cmd(patch_full_cmd, shell=True, cwd=pristine_source_path) if result != 0: logging.debug(f"[DEBUG] cmd {patch_cmd} failed for patching.") continue patched = True break if not patched: logging.error(f"[ERROR] Unable to patch. Make sure to 'patch' or 'git' is installed.") return patched except Exception as e: logging.fatal(f'[FATAL] Error applying patch file {patch_file} : {e}') return False def configure_build(temp_folder): try: pristine_source_path = str((temp_folder / SOURCE_NAME).resolve()) if platform_name == PLATFORM_WINDOWS: # Windows does not have a configure, instead it will use a modified visualc.mak directly # Copy the modified visualc.mak file to the patched source directory for the subsequent build step src_visualc_mak = SCRIPT_PATH / 'visualc.mak' dst_visualc_mak = temp_folder / SOURCE_NAME / 'src' / 'visualc.mak' shutil.copyfile(str(src_visualc_mak.resolve()), str(dst_visualc_mak.resolve())) else: if platform_name == PLATFORM_MAC: # For mac, we need to disable the 'implicit-function-declaration' or else the build will fail env_copy = os.environ.copy() env_copy['CFLAGS'] = '-Wno-implicit-function-declaration' else: env_copy = None # Mac and Linux use the built in ./configure command if execute_cmd(['./configure', '--with-pic', '--enable-mcpplib'], cwd=pristine_source_path, suppress_std_err=True, env=env_copy) != 0: logging.fatal(f'[ERROR] Error configuring build.') return False return True except Exception as e: logging.fatal(f'[FATAL] Error configuring build : {e}') return False def build_from_source(temp_folder): try: if platform_name == PLATFORM_WINDOWS: # Windows will use a precreated visualc.mak file instead of configure/make. source_working_path = str((temp_folder / SOURCE_NAME / 'src').resolve()) build_cmds = [ ['nmake', '/f', 'visualc.mak', 'COMPILER=MSC'], ['nmake', '/f', 'visualc.mak', 'COMPILER=MSC', 'MCPP_LIB=1', 'mcpplib'] ] for build_cmd in build_cmds: if execute_cmd(build_cmd, cwd=source_working_path) != 0: logging.fatal(f'[ERROR] Error building from source.') return False else: # Mac/Linux will use 'make' to build pristine_source_path = str((temp_folder / SOURCE_NAME).resolve()) result = execute_cmd(['make'], cwd=pristine_source_path, suppress_std_err=True) if result != 0: logging.fatal(f'[ERROR] Error building from source.') return False return True except Exception as e: logging.fatal(f'[FATAL] Error building from source : {e}') return False def copy_build_artifacts(temp_folder): # Copying LICENSE, headers and libs source_path = temp_folder / SOURCE_NAME target_mcpp_root = TARGET_3PP_PACKAGE_FOLDER / 'mcpp' file_copy_tuples = [ (source_path / 'LICENSE', target_mcpp_root), (source_path / 'src' / 'mcpp_lib.h', target_mcpp_root / 'include'), (source_path / 'src' / 'mcpp_out.h', target_mcpp_root / 'include') ] if platform_name in ('linux', 'linux-aarch64'): file_copy_tuples.extend([ (source_path / 'src' / '.libs' / 'libmcpp.a', target_mcpp_root / 'lib'), (source_path / 'src' / '.libs' / 'libmcpp.so.0.3.0', target_mcpp_root / 'lib'), (source_path / 'src' / '.libs' / 'mcpp', target_mcpp_root / 'lib') ]) elif platform_name == 'mac': file_copy_tuples.extend([ (source_path / 'src' / '.libs' / 'libmcpp.a', target_mcpp_root / 'lib'), (source_path / 'src' / '.libs' / 'libmcpp.0.3.0.dylib', target_mcpp_root / 'lib'), (source_path / 'src' / '.libs' / 'mcpp', target_mcpp_root / 'lib') ]) elif platform_name == 'windows': file_copy_tuples.extend([ (source_path / 'src' / 'mcpp0.dll', target_mcpp_root / 'lib'), (source_path / 'src' / 'mcpp0.lib', target_mcpp_root / 'lib'), (source_path / 'src' / 'mcpp.exe', target_mcpp_root / 'lib') ]) for file_copy_tuple in file_copy_tuples: src = file_copy_tuple[0] dst = file_copy_tuple[1] if not src.is_file(): logging.error(f'[ERROR] Missing source file {str(src)}') return False if not dst.is_dir(): os.makedirs(str(dst.resolve())) shutil.copy2(str(src), str(dst)) dst_lib_folder = target_mcpp_root / 'lib' if platform_name in ('linux', 'linux-aarch64'): base_shared_lib_name = 'libmcpp.so.0.3.0' symlinks = ['libmcpp.so.0', 'libmcpp.so'] elif platform_name == 'mac': base_shared_lib_name = 'libmcpp.0.3.0.dylib' symlinks = ['libmcpp.0.dylib', 'libmcpp.dylib'] else: base_shared_lib_name = None symlinks = None if base_shared_lib_name and symlinks: for symlink in symlinks: execute_cmd(['ln', '-s', base_shared_lib_name, symlink], cwd=str(dst_lib_folder)) return True def create_3PP_package(temp_folder, package_label): if TARGET_3PP_PACKAGE_FOLDER.is_dir(): shutil.rmtree(str(TARGET_3PP_PACKAGE_FOLDER.resolve()), ignore_errors=True) os.makedirs(str(TARGET_3PP_PACKAGE_FOLDER.resolve()), exist_ok=True) # Generate the find cmake file from the template file find_cmake_template_file = SCRIPT_PATH / f'Findmcpp.cmake.template' assert find_cmake_template_file.is_file(), f"Missing template file {find_cmake_template_file}" find_cmake_template_file_content = find_cmake_template_file.read_text("UTF-8", "ignore") template_env = { "MCPP_SHARED_LIB": shared_lib_name, "MCPP_STATIC_LIB": static_lib_name } find_cmake_content = string.Template(find_cmake_template_file_content).substitute(template_env) dst_find_cmake = TARGET_3PP_PACKAGE_FOLDER / 'Findmcpp.cmake' dst_find_cmake.write_text(find_cmake_content) # Generate the PackageInfo package_info_content = f''' {{ "PackageName" : "{package_label}-{platform_name}", "URL" : "{SOURCEFORGE_URL}", "License" : "custom", "LicenseFile" : "mcpp/LICENSE" }} ''' package_info_target = TARGET_3PP_PACKAGE_FOLDER / 'PackageInfo.json' logging.debug(f'[DEBUG] Generating {package_info_target}') package_info_target.write_text(package_info_content) return copy_build_artifacts(temp_folder) def main(): parser = argparse.ArgumentParser(description="Script to build the O3DE complaint 3rd Party Package version of the mcpp open source project.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=MCPP_DETAIL) parser.add_argument( 'package_label', help="The package name and revision" ) parser.add_argument( '--debug', help="Enable debug messages", action="store_true" ) args = parser.parse_args() logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if args.debug else logging.INFO) logging.info("Preparing temp working folder") temp_folder = SCRIPT_PATH / 'temp' if not prepare_temp_folder(temp_folder): return False logging.info("Downloading from sourceforge") if not download_from_source_forge(SOURCEFORGE_DOWNLOAD_URL, temp_folder): return False logging.info("Extracting source tarball") if not extract_tarfile(temp_folder): return False logging.info("Initializing temporary git repo") if not init_git_repo(temp_folder): return False logging.info("Apply Patch File") if not apply_patch(temp_folder, PATCH_FILE): return False logging.info("Configuring Build") if not configure_build(temp_folder): return False logging.info("Building from source") if not build_from_source(temp_folder): return False logging.info("Creating 3PP Target") if not create_3PP_package(temp_folder, args.package_label): return False # If successful, delete the temp folder if temp_folder.exists(): shutil.rmtree(str(temp_folder.resolve()), ignore_errors=True) logging.info("MCPP Package complete") return True if __name__ == '__main__': start = time.time() result = main() elapsed = time.time() - start hour = int(elapsed // 3600) minute = int((elapsed - 3600*hour) // 60) seconds = int((elapsed - 3600*hour - 60*minute)) logging.info(f'Total time {hour}:{minute:02d}:{seconds:02d}') if result: exit(0) else: exit(1)