# # Copyright (c) Contributors to the Open 3D Engine Project. # For complete copyright and license terms please see the LICENSE at the root of https://www.github.com/o3de/o3de. # # SPDX-License-Identifier: Apache-2.0 OR MIT # # import argparse import pathlib import logging import os import time import glob import sys import platform from o3de.validation import valid_o3de_project_json, valid_o3de_engine_json from queue import Queue, Empty from threading import Thread from typing import List from subprocess import Popen, PIPE logger = logging.getLogger('o3de.mps_export') LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s' logging.basicConfig(format=LOG_FORMAT) # This is an export script for MPS # this has to be a complete standalone script, b/c project export doesnt exist in main branch yet # View the argparse parameters for options available. An example invocation: # @ # > python\python.cmd \ExportScripts\export_standalone_monolithic.py -pp -ep -bnmt -out -a -aof zip def enqueue_output(out, queue): for line in iter(out.readline, b''): queue.put(line) out.close() def safe_kill_processes(*processes: List[Popen], process_logger: logging.Logger = None) -> None: """ Kills a given process without raising an error :param processes: An iterable of processes to kill :param process_logger: (Optional) logger to use """ def on_terminate(proc) -> None: try: process_logger.info(f"process '{proc.args[0]}' with PID({proc.pid}) terminated with exit code {proc.returncode}") except Exception: # purposefully broad process_logger.error("Exception encountered with termination request, with stacktrace:", exc_info=True) if not process_logger: process_logger = logger for proc in processes: try: process_logger.info(f"Terminating process '{proc.args[0]}' with PID({proc.pid})") proc.kill() except Exception: # purposefully broad process_logger.error("Unexpected exception ignored while terminating process, with stacktrace:", exc_info=True) try: for proc in processes: proc.wait(timeout=30) on_terminate(proc) except Exception: # purposefully broad process_logger.error("Unexpected exception while waiting for processes to terminate, with stacktrace:", exc_info=True) class CLICommand(object): """ CLICommand is an interface for storing CLI commands as list of string arguments to run later in a script. A current working directory, pre-existing OS environment, and desired logger can also be specified. To execute a command, use the run() function. This class is responsible for starting a new process, polling it for updates and logging, and safely terminating it. """ def __init__(self, args: list, cwd: pathlib.Path, logger: logging.Logger, env: os._Environ=None) -> None: self.args = args self.cwd = cwd self.env = env self.logger = logger self._stdout_lines = [] self._stderr_lines = [] @property def stdout_lines(self) -> List[str]: """The result of stdout, separated by newlines.""" return self._stdout_lines @property def stdout(self) -> str: """The result of stdout, as a single string.""" return "\n".join(self._stdout_lines) @property def stderr_lines(self) -> List[str]: """The result of stderr, separated by newlines.""" return self._stderr_lines @property def stderr(self) -> str: """The result of stderr, as a single string.""" return "\n".join(self._stderr_lines) def _poll_process(self, process, queue) -> None: # while process is not done, read any log lines coming from subprocess while process.poll() is None: #handle readline in a non-blocking manner try: line = queue.get_nowait() except Empty: pass else: # got line if not line: break log_line = line.decode('utf-8', 'ignore') self._stdout_lines.append(log_line) self.logger.info(log_line) def _cleanup_process(self, process, queue) -> str: # flush remaining log lines while not queue.empty(): try: line = queue.get_nowait() except Empty: pass else: if not line: break log_line = line.decode('utf-8', 'ignore') self._stdout_lines.append(log_line) self.logger.info(log_line) stderr = process.stderr.read() safe_kill_processes(process, process_logger = self.logger) return stderr def run(self) -> int: """ Takes the arguments specified during CLICommand initialization, and opens a new subprocess to handle it. This function automatically manages polling the process for logs, error reporting, and safely cleaning up the process afterwards. :return return code on success or failure """ ret = 1 try: with Popen(self.args, cwd=self.cwd, env=self.env, stdout=PIPE, stderr=PIPE) as process: self.logger.info(f"Running process '{self.args[0]}' with PID({process.pid}): {self.args}") q = Queue() t = Thread(target=enqueue_output, args=(process.stdout, q)) t.daemon = True t.start() process.stdout.flush() self._poll_process(process, q) stderr = self._cleanup_process(process, q) ret = process.returncode # print out errors if there are any if stderr: # bool(ret) --> if the process returns a FAILURE code (>0) logger_func = self.logger.error if bool(ret) else self.logger.warning err_txt = stderr.decode('utf-8', 'ignore') logger_func(err_txt) self._stderr_lines = err_txt.split("\n") except Exception as err: self.logger.error(err) raise err return ret # Helper API def process_command(args: list, cwd: pathlib.Path = None, env: os._Environ = None) -> int: """ Wrapper for subprocess.Popen, which handles polling the process for logs, reacting to failure, and cleaning up the process. :param args: A list of space separated strings which build up the entire command to run. Similar to the command list of subprocess.Popen :param cwd: (Optional) The desired current working directory of the command. Useful for commands which require a differing starting environment. :param env: (Optional) Environment to use when processing this command. :return the exit code of the program that is run or 1 if no arguments were supplied """ if len(args) == 0: logging.error("function `process_command` must be supplied a non-empty list of arguments") return 1 return CLICommand(args, cwd, logging.getLogger(), env=env).run() # EXPORT SCRIPT STARTS HERE! if __name__ == "__main__": parser = argparse.ArgumentParser(prog='Exporter for MultiplayerSample as a standalone build', description = "Exports O3DE's MultiplayerSample to the desired output directory with release layout. " "In order to use this script, the engine and project must be setup and registered beforehand. " "See this example on the MPS Github page: " "https://github.com/o3de/o3de-multiplayersample/blob/development/README.md#required-step-to-compile") parser.add_argument('-pp', '--project-path', type=pathlib.Path, required=True, help='Path to the intended O3DE project.') parser.add_argument('-ep', '--engine-path', type=pathlib.Path, required=True, help='Path to the intended O3DE engine.') parser.add_argument('-out', '--output-path', type=pathlib.Path, required=True, help='Path that describes the final resulting Release Directory path location.') parser.add_argument('-cfg', '--config', type=str, default='profile', choices=['release', 'profile'], help='The CMake build configuration to use when building project binaries. If tool binaries are built with this script, they will use profile mode.') parser.add_argument('-ll', '--log-level', default='ERROR', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Set the log level") parser.add_argument('-aof', '--archive-output-format', type=str, help="Format of archive to create from the output directory", choices=["zip", "gzip", "bz2", "xz"], default="zip") parser.add_argument('-bnmt', '--build-non-mono-tools', action='store_true') parser.add_argument('-nmbp', '--non-mono-build-path', type=pathlib.Path, default=None) parser.add_argument('-mbp', '--mono-build-path', type=pathlib.Path, default=None) parser.add_argument('-nogame', '--no-game-launcher', action='store_true', help='this flag skips building the Game Launcher on a platform if not needed.') parser.add_argument('-noserver', '--no-server-launcher', action='store_true', help='this flag skips building the Server Launcher on a platform if not needed.') parser.add_argument('-nounified', '--no-unified-launcher', action='store_true', help='this flag skips building the Unified Launcher on a platform if not needed.') parser.add_argument('-pl', '--platform', type=str, default=None, choices=['pc', 'linux', 'mac']) parser.add_argument('-a', '--archive-output', action='store_true', help='This option places the final output of the build into a compressed archive') parser.add_argument('-q', '--quiet', action='store_true', help='Suppresses logging information unless an error occurs.') args = parser.parse_args() if args.quiet: logging.getLogger().setLevel(logging.ERROR) else: logging.getLogger().setLevel(args.log_level) non_mono_build_path = (args.engine_path) / 'build' / 'non_mono' if args.non_mono_build_path is None else args.non_mono_build_path mono_build_path = (args.engine_path) / 'build' / 'mono' if args.mono_build_path is None else args.mono_build_path #validation assert valid_o3de_project_json(args.project_path / 'project.json') and valid_o3de_engine_json(args.engine_path / 'engine.json') #commands are based on #https://github.com/o3de/o3de-multiplayersample/blob/development/Documentation/PackedAssetBuilds.md selected_platform = args.platform system_platform = platform.system().lower() if not selected_platform: logger.info("Platform not specified! Defaulting to Host platform...") if not system_platform: logger.error("Unable to identify host platform! Please supply the platform using '--platform'. Options are [pc, linux, mac].") sys.exit(1) if system_platform == "windows": selected_platform = "pc" elif system_platform == "linux": selected_platform = "linux" elif system_platform == "darwin": selected_platform = "mac" else: logger.error(f"MPS exporting for {system_platform} is currently unsupported! Please use either a Windows, Mac or Linux machine to build project.") sys.exit(1) logger.info(f"Project path for MPS: {args.project_path}") logger.info(f"Engine path to build MPS: {args.engine_path}") logger.info(f"Build path for non-monolithic executables: {args.non_mono_build_path}") logger.info(f"Build path for monolithic executables: {args.mono_build_path}") output_cache_path = args.output_path / 'Cache' / selected_platform output_aws_gem_path = args.output_path / 'Gems' / 'AWSCore' os.makedirs(output_cache_path, exist_ok=True) os.makedirs(output_aws_gem_path, exist_ok=True) #Build o3de-multiplayersample and the engine (non-monolithic) if args.build_non_mono_tools: process_command(['cmake', '-S', '.', '-B', str(non_mono_build_path), '-DLY_MONOLITHIC_GAME=0', f'-DLY_PROJECTS={args.project_path}'], cwd=args.engine_path) process_command(['cmake', '--build', str(non_mono_build_path), '--target', 'AssetBundler', 'AssetBundlerBatch', 'AssetProcessor', 'AssetProcessorBatch', '--config','profile'], cwd=args.engine_path) process_command(['cmake', '--build', str(non_mono_build_path), '--target', 'MultiplayerSample.Assets', '--config', 'profile'], cwd=args.engine_path) #Build monolithic game process_command(['cmake', '-S', '.', '-B', str(mono_build_path), '-DLY_MONOLITHIC_GAME=1', '-DALLOW_SETTINGS_REGISTRY_DEVELOPMENT_OVERRIDES=0', f'-DLY_PROJECTS={args.project_path}'], cwd=args.engine_path) if not args.no_game_launcher: process_command(['cmake', '--build', str(mono_build_path), '--target', 'MultiplayerSample.GameLauncher', '--config', args.config], cwd=args.engine_path) if not args.no_server_launcher: process_command(['cmake', '--build', str(mono_build_path), '--target', 'MultiplayerSample.ServerLauncher', '--config', args.config], cwd=args.engine_path) if not args.no_unified_launcher: process_command(['cmake', '--build', str(mono_build_path), '--target', 'MultiplayerSample.UnifiedLauncher', '--config', args.config], cwd=args.engine_path) #Before bundling content, make sure that the necessary executables exist asset_bundler_batch_path = non_mono_build_path / 'bin' / 'profile' / ('AssetBundlerBatch' + ('.exe' if system_platform=='windows' else '')) if not asset_bundler_batch_path.is_file(): logger.error(f"AssetBundlerBatch not found at path '{asset_bundler_batch_path}'. In order to bundle the data for MPS, this executable must be present!") logger.error("To correct this issue, do 1 of the following: " "1) Use the --build-non-mono-tools flag in the CLI parameters" "2) If you are trying to build in a project-centric fashion, please switch to engine-centric for this export script" f"3) Build AssetBundlerBatch by hand and make sure it is available at {asset_bundler_batch_path}" "4) Set the --non-mono-build-path to point at a directory which contains this executable") sys.exit(1) #Bundle content engine_asset_list_path = args.project_path / 'AssetBundling' / 'AssetLists' / f'engine_{selected_platform}.assetlist' process_command([asset_bundler_batch_path, 'assetLists','--addDefaultSeedListFiles', '--assetListFile', engine_asset_list_path, '--project-path', args.project_path, '--allowOverwrites' ], cwd=args.engine_path) game_asset_list_path = args.project_path /'AssetBundling'/'AssetLists'/ f'game_{selected_platform}.assetlist' seed_folder_path = args.project_path/'AssetBundling'/'SeedLists' game_asset_list_command = [asset_bundler_batch_path, 'assetLists', '--assetListFile', game_asset_list_path, '--seedListFile', seed_folder_path / 'BasePopcornFxSeedList.seed', '--seedListFile', seed_folder_path / 'GameSeedList.seed'] if args.config == 'profile': # Dev branch has removed the profile seed list, but it still remains in main for now. # This will be removed after the next release, when both branches are synchronized profile_seed_list_path = seed_folder_path / 'ProfileOnlySeedList.seed' if profile_seed_list_path.is_file(): game_asset_list_command += ['--seedListFile', profile_seed_list_path] game_asset_list_command += ['--seedListFile', seed_folder_path / 'VFXSeedList.seed', '--project-path', args.project_path, '--allowOverwrites'] process_command(game_asset_list_command, cwd=args.engine_path) engine_bundle_path = output_cache_path / f'engine_{selected_platform}.pak' process_command([asset_bundler_batch_path, 'bundles', '--assetListFile', engine_asset_list_path, '--outputBundlePath', engine_bundle_path, '--project-path', args.project_path, '--allowOverwrites'], cwd=args.engine_path) # This is to prevent any accidental file locking mechanism from failing subsequent bundling operations time.sleep(1) game_bundle_path = output_cache_path / f'game_{selected_platform}.pak' process_command([asset_bundler_batch_path, 'bundles', '--assetListFile', game_asset_list_path, '--outputBundlePath', game_bundle_path, '--project-path', args.project_path, '--allowOverwrites'], cwd=args.engine_path) # Create Launcher Layout Directory import shutil for file in glob.glob(str(pathlib.PurePath(mono_build_path / 'bin' / args.config / '*.*'))): shutil.copy(file, args.output_path) for file in glob.glob(str(pathlib.PurePath(mono_build_path / 'bin' / args.config / 'Gems' / 'AWSCore' / '*.*'))): shutil.copy(file, output_aws_gem_path) for file in glob.glob(str(pathlib.PurePath(args.project_path / 'launch_*.*'))): shutil.copy(file, args.output_path) # Optionally zip the layout directory if the user requests if args.archive_output: archive_name = args.output_path logger.info("Archiving output directory (this may take a while)...") shutil.make_archive(args.output_path, args.archive_output_format, root_dir = args.output_path) logger.info(f"Exporting project is complete! Release Directory can be found at {args.output_path}")