pull_and_build_from_git.py 68 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287
  1. #
  2. # Copyright (c) Contributors to the Open 3D Engine Project.
  3. # For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. #
  5. # SPDX-License-Identifier: Apache-2.0 OR MIT
  6. #
  7. #
  8. import argparse
  9. import fnmatch
  10. import glob
  11. import json
  12. import os
  13. import pathlib
  14. import platform
  15. import re
  16. import shlex
  17. import shutil
  18. import string
  19. import subprocess
  20. import sys
  21. from package_downloader import PackageDownloader
  22. from archive_downloader import download_and_verify, extract_package
  23. SCHEMA_DESCRIPTION = """
  24. Build Config Description:
  25. The build configuration (build_config.json) accepts keys that are root level only, and some keys that can be
  26. either global or target platform specific. Root level only keys are keys that define the project and cannot
  27. be different by platform, and all are required. The keys are:
  28. * package_name : The base name of the package, used for constructing the filename and folder structures
  29. * package_url : The package url that will be placed in the PackageInfo.json
  30. * package_license : The type of license that will be described in the PackageInfo.json
  31. * package_license_file : The name of the source code license file (expected at the root of the source folder pulled from git)
  32. The following keys can exist at the root level or the target-platform level:
  33. * git_url : The git clone url for the source to pull for building
  34. * git_tag : The git tag or branch to identify the branch to pull from for building
  35. * git_commit : (optional) A specific git commit to check out. This is useful for upstream repos that do not tag their releases.
  36. * src_package_url : The download URI to retrieve the source package compressed tar from
  37. * src_package_sha1 : The sha1 fingerprint of the downloaded source package compressed tar for verification
  38. ** Note: Either both git_url + git_tag/git_commit OR src_package_url + src_package_sha1 must be supplied, but not both
  39. * package_version : (required) The string to describe the package version. This string is used to build the full package name.
  40. This can be uniform for all platforms or can be set for a specific platform
  41. * prebuilt_source : (optional) If the 3rd party library files are prebuilt and accessible, then setting this key to the relative location of
  42. the folder will cause the workflow to perform copy operations into the generated target library folder directly (see
  43. 'prebuilt_args' below.
  44. * prebuild_args : (required if prebuilt_source is set) A map of target subfolders within the target 3rd party folder against a glob pattern of
  45. file(s) to copy to the target subfolders.
  46. * cmake_find_source : The name of the source Find*.cmake file that will be used in the target package
  47. that is ingested by the lumberyard 3P system.
  48. * cmake_find_template : If the find*.cmake in the target package requires template processing, then this is name of the template file that is used to
  49. generate the contents of the find*.cmake file in the target package.
  50. * Note that either 'cmake_find_source' or 'cmake_find_template' must be declared.
  51. * cmake_find_target : (required if prebuilt_source is not set) The name of the target find*.cmake file that is generated based on the template file and
  52. additional arguments (described below)
  53. * build_configs : (optional) A list of configurations to build during the build process. This is available
  54. to restrict building to a specific configuration rather than building all configurations
  55. (provided by the default value: ['Debug', 'Release'])
  56. * patch_file : (optional) Option patch file to apply to the synced source before performing a build
  57. * source_path : (optional) Option to provide a path to the project source rather than getting it from github
  58. * git_skip : (optional) Option to skip all git commands, requires source_path
  59. * cmake_src_subfolder : (optional) Some packages don't have a CMakeLists at the root and instead its in a subfolder.
  60. In this case, set this to be the relative path from the src root to the folder that
  61. contains the CMakeLists.txt.
  62. * cmake_generate_args_common : (optional) When used at the root, this provides a set of cmake arguments for generation which will
  63. apply to ALL platforms and configs (appended to cmake_generate_args).
  64. Can be overriden by a specific platform by specifying it in the platform specific section.
  65. The final args will be (cmake_generate_args || cmake_generation_args_CONFIG) + cmake_generate_args_common
  66. * cmake_build_args_common : (optional) When used at the root, provides a set of cmake arguments for building which will apply to ALL
  67. platforms and configurations.
  68. The final args will be (cmake_build_args || cmake_build_args_CONFIG) + cmake_build_args_common
  69. `cmake --build (build folder) --config config` will automatically be supplied.
  70. * extra_files_to_copy : (optional) a list of pairs or triplets.
  71. if the item is a pair, then it can be (source file, destination file) or
  72. (source directory, destination directory).
  73. Directories are always deep copied.
  74. If a triplet is specified, then the third element is another list of file patterns to ignore when copying a directory.
  75. * cmake_install_filter : Optional list of filename patterns to filter what is actually copied to the target package based on
  76. the 3rd party library's install definition. (For example, a library may install headers and static
  77. libraries when all you want in the package is just the binary executables). If omitted, then the entire
  78. install tree will be copied to the target package.
  79. This field can exist at the root but also at individual platform target level.
  80. The following keys can only exist at the target platform level as they describe the specifics for that platform.
  81. * cmake_generate_args : The cmake generation arguments (minus the build folder target or any configuration) for generating
  82. the project for the platform (for all configurations). To perform specific generation commands (i.e.
  83. for situations where the generator does not support multiple configs) the key can contain the
  84. suffix of the configuration name (cmake_generate_args_debug, cmake_generate_args_release).
  85. For common args that should apply to every config, see cmake_generate_args_common above.
  86. * cmake_build_args : Additional build args to pass to cmake during the cmake build command
  87. * custom_build_cmd : A custom build script and arguments to build from the source that was pulled from git. This is a list
  88. starting with the script to execute along with a list of optional arguments to the script. This is mutually
  89. exclusive from the cmake_generate_args and cmake_build_args options.
  90. Note: If the command is a python script, format the command with a {python} variable, for example: "{python} build_me.py"
  91. This will invoke the same python interpreter that is used to launch the build package script
  92. see the note about environment variables below.
  93. * custom_install_cmd : A custom script and arguments to run (after the custom_build_cmd) to copy and assemble the built binaries
  94. into the target package folder. This is a list starting with the script to execute along with a list of optional
  95. arguments to the script. This argument is optional. You could do the install in your custom build command instead.
  96. Note: If the command is a python script, format the command with a {python} variable, for example: "{python} install_me.py"
  97. This will invoke the same python interpreter that is used to launch the build package script
  98. see the note about environment variables below.
  99. * custom_install_json : A list of files to copy into the target package folder from the built SDK. This argument is optional.
  100. * custom_test_cmd : after making the package, it will run this and expect exit code 0
  101. this argument is optional.
  102. see the note about environment variables below.
  103. * custom_additional_compile_definitions : Any additional compile definitions to apply in the find*.cmake file for the library that will applied
  104. to targets that consume this 3P library
  105. * custom_additional_link_options : Any additional linker options to apply in the find*.cmake file for the library that will applied
  106. to targets that consume this 3P library during linking
  107. * custom_additional_libraries : Any additional dependent system library to include in the find*.cmake file for the library that will
  108. applied to targets that consume this 3P library during linking
  109. * custom_additional_template_map : Any additional custom template mappings to apply if a `cmake_find_template` was specified
  110. * depends_on_packages : list of name of 3-TUPLES of [package name, package hash, subfolder] that 'find' files live in]
  111. [ ["zlib-1.5.3-rev5", "some hash", ""],
  112. ["some other package", "some other hash", "subfoldername"],
  113. ...
  114. ]
  115. that we need to download and use).
  116. - note that we don't check recursively - you must name your recursive deps!
  117. - The packages must be on a public CDN or locally tested with FILE:// - it uses env var
  118. "LY_PACKAGE_SERVER_URLS" which can be a semicolon seperated list of places to try.
  119. - The packages unzip path + subfolder is added to CMAKE_MODULE_PATH if you use cmake commands.
  120. - Otherwise you can use DOWNLOADED_PACKAGE_FOLDERS env var in your custom script and set
  121. - CMAKE_MODULE_PATH to be that value, yourself.
  122. - The subfolder can be empty, in which case the root of the package will be used.
  123. * additional_download_packages : list of archived package files to download and extract for use in any custom build script. The packages will
  124. be extracted to the working temp folder. The list will be a list of 3-TUPLES of
  125. [full_download_url, file hash, hash algorithm] where:
  126. full_download_url - The full download URL of the package to download
  127. file hash - The hex-string of the fingerprint to validate the download with. If this is left blank, no validation
  128. will be done, instead it will be calculated on the downloaded package and printed to the console.
  129. hash algorithm - The hash algorithm to use to calculate the file hash.
  130. Note about environment variables:
  131. When custom commands are issued (build, install, and test), the following environment variables will be set
  132. for the process:
  133. PACKAGE_ROOT = root of the package being made (where PackageInfo.json is generated/copied)
  134. TARGET_INSTALL_ROOT = $PACKAGE_ROOT/$PACKAGE_NAME - usually where you target cmake install to
  135. TEMP_FOLDER = the temp folder. This folder usually has subfolder 'build' and 'src'
  136. PYTHON_BINARY = the path to the python binary that launched the build script. This can be useful if
  137. one of the custom build/install scripts (e.g. my_script.sh/.cmd) want to invoke
  138. a python script using the same python executable that launched the build.
  139. DOWNLOADED_PACKAGE_FOLDERS = semicolon seperated list of abs paths to each downloaded package Find folder.
  140. - usually used to set CMAKE_MODULE_PATH so it can find the packages.
  141. - unset if there are no dependencies declared
  142. Note that any of the above environment variables that contain paths will use system native slashes for script
  143. compatibility, and may need to be converted to forward slash in your script on windows
  144. if you feed it to cmake.
  145. Also note that the working directory for all custom commands will the folder containing the build_config.json file.
  146. The general layout of the build_config.json file is as follows:
  147. {
  148. ${root level keys}
  149. ${global keys}
  150. "Platforms": {
  151. ${Host Platforms}: {
  152. ${Target Platform}: {
  153. ${platform specific general keys}
  154. ${platform specific required keys}
  155. }
  156. }
  157. }
  158. }
  159. """
  160. # The current path of this script, expected to be under '3rdPartySource/Scripts'
  161. CURRENT_PATH = pathlib.Path(os.path.dirname(__file__)).resolve()
  162. # Expected package-system folder as the parent of this folder
  163. PACKAGE_SYSTEM_PATH = CURRENT_PATH.parent.parent / 'package-system'
  164. assert PACKAGE_SYSTEM_PATH.is_dir(), "Missing package-system folder, make sure it is synced from source control"
  165. # Some platforms required environment variables to be set before the build, create the appropriate pattern to search for it
  166. if platform.system() == 'Windows':
  167. ENV_PATTERN = re.compile(r"(%([a-zA-Z0-9_]*)%)")
  168. else:
  169. ENV_PATTERN = re.compile(r"($([a-zA-Z0-9_]*))")
  170. DEFAULT_BUILD_CONFIG_FILENAME = "build_config.json"
  171. class BuildError(Exception):
  172. """
  173. Manage Package Build specific exceptions
  174. """
  175. pass
  176. class PackageInfo(object):
  177. """
  178. This class manages general information for the package based on the build config and target platform
  179. information. It does not manage the actual cmake commands
  180. """
  181. PACKAGE_INFO_TEMPLATE = """{
  182. "PackageName" : "$package_name-$package_version-$platform_name",
  183. "URL" : "$package_url",
  184. "License" : "$package_license",
  185. "LicenseFile" : "$package_name/$package_license_file"
  186. }
  187. """
  188. def __init__(self, build_config, target_platform_name, target_platform_config):
  189. """
  190. Initialize the PackageInfo
  191. :param build_config: The entire build configuration dictionary (from the build config json file)
  192. :param target_platform_name: The target platform name that is being packaged for
  193. :param target_platform_config: The target platform configuration (from the build configuration dictionary)
  194. """
  195. self.platform_name = target_platform_name
  196. try:
  197. self.package_name = build_config["package_name"]
  198. self.package_url = build_config["package_url"]
  199. self.package_license = build_config["package_license"]
  200. self.package_license_file = build_config["package_license_file"]
  201. except KeyError as e:
  202. raise BuildError(f"Invalid build config. Missing required key : {str(e)}")
  203. def _get_value(value_key, required=True, default=None):
  204. result = target_platform_config.get(value_key, build_config.get(value_key, default))
  205. if required and result is None:
  206. raise BuildError(f"Required key '{value_key}' not found in build config")
  207. return result
  208. self.git_url = _get_value("git_url", required=False)
  209. self.git_tag = _get_value("git_tag", required=False)
  210. self.src_package_url = _get_value("src_package_url", required=False)
  211. self.src_package_sha1 = _get_value("src_package_sha1", required=False)
  212. if not self.git_url and not self.src_package_url:
  213. raise BuildError(f"Either 'git_url' or 'src_package_url' must be provided for the source in the build config.")
  214. if self.git_url and self.src_package_url:
  215. raise BuildError(f"Only 'git_url' or 'src_package_url' can be specified, not both. Both were specified in this build config.")
  216. if self.git_url and not self.git_tag:
  217. raise BuildError(f"Missing 'git_tag' entry for the git repo {self.git_url} in the build config.")
  218. if self.src_package_url and not self.src_package_sha1:
  219. raise BuildError(f"Missing 'src_package_sha1' entry for the source package at {self.src_package_url} in the build config.")
  220. self.package_version = _get_value("package_version")
  221. self.patch_file = _get_value("patch_file", required=False)
  222. self.git_commit = _get_value("git_commit", required=False)
  223. self.cmake_find_template = _get_value("cmake_find_template", required=False)
  224. self.cmake_find_source = _get_value("cmake_find_source", required=False)
  225. self.cmake_find_target = _get_value("cmake_find_target")
  226. self.cmake_find_template_custom_indent = _get_value("cmake_find_template_custom_indent", default=1)
  227. self.additional_src_files = _get_value("additional_src_files", required=False)
  228. self.depends_on_packages = _get_value("depends_on_packages", required=False)
  229. self.additional_download_packages = _get_value("additional_download_packages", required=False)
  230. self.cmake_src_subfolder = _get_value("cmake_src_subfolder", required=False)
  231. self.cmake_generate_args_common = _get_value("cmake_generate_args_common", required=False)
  232. self.cmake_build_args_common = _get_value("cmake_build_args_common", required=False)
  233. self.build_configs = _get_value("build_configs", required=False, default=['Debug', 'Release'])
  234. self.extra_files_to_copy = _get_value("extra_files_to_copy", required=False)
  235. self.cmake_install_filter = _get_value("cmake_install_filter", required=False, default=[])
  236. self.custom_toolchain_file = _get_value("custom_toolchain_file", required=False)
  237. if self.cmake_find_template and self.cmake_find_source:
  238. raise BuildError("Bad build config file. 'cmake_find_template' and 'cmake_find_source' cannot both be set in the configuration.")
  239. if not self.cmake_find_template and not self.cmake_find_source:
  240. raise BuildError("Bad build config file. 'cmake_find_template' or 'cmake_find_source' must be set in the configuration.")
  241. def write_package_info(self, install_path):
  242. """
  243. Write to the target 'PackageInfo.json' file for the package
  244. :param install_path: The folder to write the file to
  245. """
  246. package_info_target_file = install_path / "PackageInfo.json"
  247. if package_info_target_file.is_file():
  248. package_info_target_file.unlink()
  249. package_info_env = {
  250. 'package_name': self.package_name,
  251. 'package_version': self.package_version,
  252. 'platform_name': self.platform_name.lower(),
  253. 'package_url': self.package_url,
  254. 'package_license': self.package_license,
  255. 'package_license_file': os.path.basename(self.package_license_file)
  256. }
  257. package_info_content = string.Template(PackageInfo.PACKAGE_INFO_TEMPLATE).substitute(package_info_env)
  258. package_info_target_file.write_text(package_info_content)
  259. def subp_args(args):
  260. """
  261. According to subcommand, when using shell=True, its recommended not to pass in an argument list but the full command line as a single string.
  262. That means in the argument list in the configuration make sure to provide the proper escapements or double-quotes for paths with spaces
  263. :param args: The list of arguments to transform
  264. """
  265. arg_string = " ".join([arg for arg in args])
  266. print(f"Command: {arg_string}")
  267. return arg_string
  268. def validate_git():
  269. """
  270. If make sure git is available
  271. :return: String describing the version of the detected git
  272. """
  273. call_result = subprocess.run(subp_args(['git', '--version']), shell=True, capture_output=True)
  274. if call_result.returncode != 0 and call_result.returncode != 1:
  275. raise BuildError("Git is not installed on the default path. Make sure its installed")
  276. version_result = call_result.stdout.decode('UTF-8', 'ignore').strip()
  277. return version_result
  278. def validate_cmake(cmake_path):
  279. """
  280. Make sure that the cmake command being used is available and confirm the version
  281. :return: String describing the version of cmake
  282. """
  283. call_result = subprocess.run(subp_args([cmake_path, '--version']), shell=True, capture_output=True)
  284. if call_result.returncode != 0:
  285. raise BuildError(f"Unable to detect CMake ({cmake_path})")
  286. version_result_lines = call_result.stdout.decode('UTF-8', 'ignore').split('\n')
  287. version_result = version_result_lines[0]
  288. print(f"Detected CMake: {version_result}")
  289. return cmake_path
  290. def validate_patch():
  291. """
  292. Make sure patch is installed and on the default path
  293. :return: String describing the version of patch
  294. """
  295. call_result = subprocess.run(subp_args(['patch', '--version']), shell=True, capture_output=True)
  296. if call_result.returncode != 0:
  297. raise BuildError("'Patch' is not installed on the default path. Make sure its installed")
  298. version_result_lines = call_result.stdout.decode('UTF-8', 'ignore').split('\n')
  299. version_result = version_result_lines[0]
  300. return version_result
  301. def create_folder(folder):
  302. """
  303. Handles error checking and messaging for creating a tree of folders.
  304. It is assumed that it is okay if the folder exists, but not okay if the
  305. folder is a file.
  306. """
  307. # wrap it up in a Path so that if a string is passed in, this still works.
  308. path_folder = pathlib.Path(folder).resolve(strict=False)
  309. if path_folder.is_file():
  310. print(f"create_folder expected a folder but found a file: {path_folder}")
  311. path_folder.mkdir(parents=True, exist_ok=True)
  312. def delete_folder(folder):
  313. """
  314. Use the system's remove folder command instead of os.rmdir().
  315. This function does various checks before trying, to avoid having to do those
  316. checks over and over in code.
  317. """
  318. # wrap it up in a Path so that if a string is passed in, this still works.
  319. path_folder = pathlib.Path(folder).resolve(strict=False)
  320. if path_folder.is_file():
  321. print(f"Expected a folder, but found a file: {path_folder}")
  322. if not path_folder.is_dir():
  323. return
  324. if platform.system() == 'Windows':
  325. call_result = subprocess.run(subp_args(['rmdir', '/Q', '/S', str(path_folder)]),
  326. shell=True,
  327. capture_output=True,
  328. cwd=str(path_folder.parent.resolve()))
  329. else:
  330. call_result = subprocess.run(subp_args(['rm', '-rf', str(path_folder)]),
  331. shell=True,
  332. capture_output=True,
  333. cwd=str(path_folder.parent.resolve()))
  334. if call_result.returncode != 0:
  335. raise BuildError(f"Unable to delete folder {str(path_folder)}: {str(call_result.stderr)}")
  336. def validate_args(input_args):
  337. """
  338. Validate and make sure that if any environment variables are passed into the argument that the environment variable is actually set
  339. """
  340. if input_args:
  341. for arg in input_args:
  342. match_env = ENV_PATTERN.search(arg)
  343. if not match_env:
  344. continue
  345. env_var_name = match_env.group(2)
  346. if not env_var_name:
  347. continue
  348. env_var_value = os.environ.get(env_var_name)
  349. if not env_var_value:
  350. raise BuildError(f"Required environment variable '{env_var_name}' not set")
  351. return input_args
  352. class BuildInfo(object):
  353. """
  354. This is the Build management class that will perform the entire build from source and preparing a folder for packaging
  355. """
  356. def __init__(self, package_info, platform_config, base_folder, build_folder, package_install_root,
  357. cmake_command, clean_build, cmake_find_template,
  358. cmake_find_source, prebuilt_source, prebuilt_args, src_folder, skip_git):
  359. """
  360. Initialize the Build management object with information needed
  361. :param package_info: The PackageInfo object constructed from the build config
  362. :param platform_config: The target platform configuration from the build config dictionary
  363. :param base_folder: The base folder where the build_config exists
  364. :param build_folder: The root folder to build into
  365. :param package_install_root: The root of the package folder where the new package will be assembled
  366. :param cmake_command: The cmake executable command to use for cmake
  367. :param clean_build: Option to clean any existing build folder before proceeding
  368. :param cmake_find_template: The template for the find*.cmake generated file
  369. :param cmake_find_source: The source file for the find*.cmake generated file
  370. :param prebuilt_source: If provided, the git fetch / build flow will be replaced with a copy from a prebuilt folder
  371. :param prebuilt_args: If prebuilt_source is provided, then this argument is required to specify the copy rules to assemble the package from the prebuilt package
  372. :param src_folder: Path to the source code / where to clone the git repo.
  373. :param skip_git: If true skip all git interaction and .
  374. """
  375. assert (cmake_find_template is not None and cmake_find_source is None) or \
  376. (cmake_find_template is None and cmake_find_source is not None), "Either cmake_find_template or cmake_find_source must be set, but not both"
  377. self.package_info = package_info
  378. self.platform_config = platform_config
  379. self.cmake_command = cmake_command
  380. self.base_folder = base_folder
  381. self.base_temp_folder = build_folder
  382. self.src_folder = src_folder
  383. self.build_folder = self.base_temp_folder / "build"
  384. self.package_install_root = package_install_root / f"{package_info.package_name}-{package_info.platform_name.lower()}"
  385. self.build_install_folder = self.package_install_root / package_info.package_name
  386. self.clean_build = clean_build
  387. self.cmake_find_template = cmake_find_template
  388. self.cmake_find_source = cmake_find_source
  389. self.build_configs = platform_config.get('build_configs', package_info.build_configs)
  390. self.prebuilt_source = prebuilt_source
  391. self.prebuilt_args = prebuilt_args
  392. self.skip_git = skip_git
  393. # Prepare any cmake_find_template format parameters
  394. def _build_list_str(indent, key):
  395. list_items = self.platform_config.get(key, [])
  396. indented_list_items = []
  397. for list_item in list_items:
  398. indented_list_items.append(f'{" "*(indent*4)}{list_item}')
  399. return '\n'.join(indented_list_items)
  400. cmake_find_template_def_ident_level = package_info.cmake_find_template_custom_indent
  401. self.cmake_template_env = {
  402. "CUSTOM_ADDITIONAL_COMPILE_DEFINITIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_compile_definitions'),
  403. "CUSTOM_ADDITIONAL_LINK_OPTIONS": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_link_options'),
  404. "CUSTOM_ADDITIONAL_LIBRARIES": _build_list_str(cmake_find_template_def_ident_level, 'custom_additional_libraries')
  405. }
  406. # Apply any custom cmake template variable parameters
  407. custom_additional_template_map = self.platform_config.get("custom_additional_template_map", {})
  408. if custom_additional_template_map:
  409. # Validate that the custom map does not include reserved template variables
  410. reserved_keys = self.cmake_template_env.keys()
  411. for custom_template_variable in custom_additional_template_map.items():
  412. if custom_template_variable[0] in reserved_keys:
  413. raise BuildError(f"Invalid entry in 'custom_additional_template_map' build config. Reserved word '{custom_template_variable[0]}' not permitted")
  414. self.cmake_template_env[custom_template_variable[0]] = custom_template_variable[1]
  415. def clone_to_local(self):
  416. """
  417. Perform a clone to the local temp folder
  418. """
  419. print(f"Cloning {self.package_info.package_name}/{self.package_info.git_tag} to {str(self.src_folder.absolute())}")
  420. working_dir = str(self.src_folder.parent.absolute())
  421. relative_src_dir = self.src_folder.name
  422. clone_cmd = ['git',
  423. 'clone',
  424. '--single-branch',
  425. '--recursive',
  426. '--branch',
  427. self.package_info.git_tag,
  428. self.package_info.git_url,
  429. relative_src_dir]
  430. clone_result = subprocess.run(subp_args(clone_cmd),
  431. shell=True,
  432. capture_output=True,
  433. cwd=working_dir)
  434. if clone_result.returncode != 0:
  435. raise BuildError(f"Error cloning from GitHub: {clone_result.stderr.decode('UTF-8', 'ignore')}")
  436. if self.package_info.git_commit is not None:
  437. # Allow the package to specify a specific commit to check out. This is useful for upstream repos that do
  438. # not tag their releases.
  439. checkout_result = subprocess.run(
  440. ['git', 'checkout', self.package_info.git_commit],
  441. capture_output=True,
  442. cwd=self.src_folder)
  443. if checkout_result.returncode != 0:
  444. raise BuildError(f"Error checking out {self.package_info.git_commit}: {checkout_result.stderr.decode('UTF-8', 'ignore')}")
  445. def prepare_temp_folders(self):
  446. """
  447. Prepare the temp folders for cloning, building, and local installing
  448. """
  449. # Always clean the target package install folder to prevent stale files from being included
  450. delete_folder(self.package_install_root)
  451. delete_folder(self.build_install_folder)
  452. if self.clean_build:
  453. delete_folder(self.build_folder)
  454. # some installs use a working temp folder as an intermediate, clean that too:
  455. working_install_folder = self.base_temp_folder / 'working_install'
  456. delete_folder(working_install_folder)
  457. create_folder(self.build_folder)
  458. create_folder(self.package_install_root)
  459. create_folder(self.build_install_folder)
  460. create_folder(working_install_folder)
  461. def sync_source(self):
  462. """
  463. Sync the 3rd party from its git source location (either cloning if its not there or syncing)
  464. """
  465. if self.skip_git:
  466. return
  467. if self.package_info.git_url:
  468. # Validate Git is installed
  469. git_version = validate_git()
  470. print(f"Detected Git: {git_version}")
  471. # Sync to the source folder
  472. if self.src_folder.is_dir():
  473. print(f"Checking git status of path '{self.src_folder}' ...")
  474. git_status_cmd = ['git', 'status', '-s']
  475. call_result = subprocess.run(subp_args(git_status_cmd),
  476. shell=True,
  477. capture_output=True,
  478. cwd=str(self.src_folder.resolve()))
  479. # If any error, this is not a valid git folder, proceed with cloning
  480. if call_result.returncode != 0:
  481. print(f"Path '{self.src_folder}' is not a valid git folder. Deleting and re-cloning...")
  482. # Not a valid git folder, okay to remove and re-clone
  483. delete_folder(self.src_folder)
  484. self.clone_to_local()
  485. else:
  486. # If this is a valid git folder, check if the patch was applied or if the source was
  487. # altered.
  488. if len(call_result.stdout.decode('utf-8', 'ignore')):
  489. # If anything changed, then restore the entire source tree
  490. print(f"Path '{self.src_folder}' was modified. Restoring...")
  491. git_restore_cmd = ['git', 'restore', '--recurse-submodules', ':/']
  492. call_result = subprocess.run(subp_args(git_restore_cmd),
  493. shell=True,
  494. capture_output=False,
  495. cwd=str(self.src_folder.resolve()))
  496. if call_result.returncode != 0:
  497. # If we cannot restore through git, then delete the folder and re-clone
  498. print(f"Unable to restore {self.src_folder}. Deleting and re-cloning...")
  499. delete_folder(self.src_folder)
  500. self.clone_to_local()
  501. # Do a re-pull
  502. git_pull_cmd = ['git',
  503. 'pull']
  504. call_result = subprocess.run(subp_args(git_pull_cmd),
  505. shell=True,
  506. capture_output=True,
  507. cwd=str(self.src_folder.resolve()))
  508. if call_result.returncode != 0:
  509. raise BuildError(f"Error pulling source from GitHub: {call_result.stderr.decode('UTF-8', 'ignore')}")
  510. else:
  511. self.clone_to_local()
  512. elif self.package_info.src_package_url:
  513. downloaded_package_file = download_and_verify(src_url=self.package_info.src_package_url,
  514. src_zip_hash=self.package_info.src_package_sha1,
  515. src_zip_hash_algorithm="sha1",
  516. target_folder=self.base_temp_folder)
  517. extracted_package_path = extract_package(src_package_file=downloaded_package_file,
  518. target_folder=self.src_folder.resolve())
  519. else:
  520. raise BuildError(f"Missing both 'git_url' and 'src_package_url' from the build config")
  521. if self.package_info.additional_src_files:
  522. for additional_src in self.package_info.additional_src_files:
  523. additional_src_path = self.base_folder / additional_src
  524. if not additional_src_path.is_file():
  525. raise BuildError(f"Invalid additional src file: : {additional_src}")
  526. additional_tgt_path = self.src_folder / additional_src
  527. if additional_tgt_path.is_file():
  528. additional_tgt_path.unlink()
  529. shutil.copy2(str(additional_src_path), str(additional_tgt_path))
  530. # Check/Validate the license file from the package, and copy over to install path
  531. if self.package_info.package_license_file:
  532. package_license_src = self.src_folder / self.package_info.package_license_file
  533. if not package_license_src.is_file():
  534. package_license_src = self.src_folder / os.path.basename(self.package_info.package_license_file)
  535. if not package_license_src.is_file():
  536. raise BuildError(f"Invalid/missing license file '{self.package_info.package_license_file}' specified in the build config.")
  537. license_file_content = package_license_src.read_text("UTF-8", "ignore")
  538. if not len(license_file_content):
  539. raise BuildError(f"license file {str(self.package_info.package_license_file)} is empty. Is this a valid license file?")
  540. target_license_copy = self.build_install_folder / os.path.basename(package_license_src)
  541. if target_license_copy.is_file():
  542. target_license_copy.unlink()
  543. shutil.copy2(str(package_license_src), str(target_license_copy))
  544. print(f"Copied license file from {package_license_src} to {target_license_copy}")
  545. # Check if there is a patch to apply
  546. if self.package_info.patch_file:
  547. patch_file_path = self.base_folder / self.package_info.patch_file
  548. if not patch_file_path.is_file():
  549. raise BuildError(f"Invalid/missing patch file '{patch_file_path}' specified in the build config.")
  550. if self.package_info.git_url:
  551. patch_cmd = ['git',
  552. 'apply',
  553. "--ignore-whitespace",
  554. str(patch_file_path.absolute())]
  555. elif self.package_info.src_package_url:
  556. patch_cmd = ['patch',
  557. '--unified',
  558. '--strip=1',
  559. f'--directory={str(self.src_folder.absolute())}',
  560. '<',
  561. str(patch_file_path.absolute())]
  562. patch_result = subprocess.run(subp_args(patch_cmd),
  563. shell=True,
  564. capture_output=True,
  565. cwd=str(self.src_folder.absolute()))
  566. if patch_result.returncode != 0:
  567. raise BuildError(f"Error Applying patch {str(patch_file_path.absolute())}: {patch_result.stderr.decode('UTF-8', 'ignore')}")
  568. # Check if there are any package dependencies.
  569. if self.package_info.depends_on_packages:
  570. for package_name, package_hash, _ in self.package_info.depends_on_packages:
  571. temp_packages_folder = self.base_temp_folder
  572. if PackageDownloader.ValidateUnpackedPackage(package_name, package_hash, str(temp_packages_folder)):
  573. print(f"Package {package_name} already downloaded")
  574. else:
  575. if not PackageDownloader.DownloadAndUnpackPackage(package_name, package_hash, str(temp_packages_folder)):
  576. raise BuildError(f"Failed to download a required dependency: {package_name}")
  577. # Check if there are any additional package dependencies to download and extract
  578. if self.package_info.additional_download_packages:
  579. print("Downloading additional packages")
  580. for package_url, package_hash, package_algorithm in self.package_info.additional_download_packages:
  581. print(f"Retrieving additional package from {package_url}")
  582. downloaded_package_file = download_and_verify(src_url=package_url,
  583. src_zip_hash=package_hash,
  584. src_zip_hash_algorithm=package_algorithm,
  585. target_folder=self.base_temp_folder)
  586. extracted_package_path = extract_package(src_package_file=downloaded_package_file,
  587. target_folder=self.base_temp_folder)
  588. def build_and_install_cmake(self):
  589. """
  590. Build and install to a local folder to prepare for packaging
  591. """
  592. is_multi_config = 'cmake_generate_args' in self.platform_config
  593. if not is_multi_config:
  594. if 'cmake_generate_args_debug' not in self.platform_config and 'cmake_generate_args_release' not in self.platform_config:
  595. raise BuildError("Invalid configuration")
  596. # Check for the optional install filter
  597. cmake_install_filter = self.platform_config.get('cmake_install_filter', self.package_info.cmake_install_filter)
  598. if cmake_install_filter:
  599. # If there is a custom install filter, then we need to install to another temp folder and copy over based on the filter rules
  600. install_target_folder = self.base_temp_folder / 'working_install'
  601. else:
  602. # Otherwise install directly to the target
  603. install_target_folder = self.build_install_folder
  604. install_target_folder = install_target_folder.resolve()
  605. can_skip_generate = False
  606. for config in self.build_configs:
  607. print(f'Configuring {config.lower()} ... ')
  608. if not can_skip_generate:
  609. cmake_generator_args = self.platform_config.get(f'cmake_generate_args_{config.lower()}')
  610. if not cmake_generator_args:
  611. cmake_generator_args = self.platform_config.get('cmake_generate_args')
  612. # Can skip generate the next time since there is only 1 unique cmake generation
  613. can_skip_generate = True
  614. # if there is a cmake_generate_args_common key in the build config, then start with that.
  615. if self.package_info.cmake_generate_args_common:
  616. cmake_generator_args = cmake_generator_args + self.package_info.cmake_generate_args_common
  617. validate_args(cmake_generator_args)
  618. cmakelists_folder = self.src_folder
  619. if self.package_info.cmake_src_subfolder:
  620. cmakelists_folder = cmakelists_folder / self.package_info.cmake_src_subfolder
  621. cmake_generate_cmd = [self.cmake_command,
  622. '-S', str(cmakelists_folder.resolve()),
  623. '-B', str(self.build_folder.name)]
  624. if self.package_info.custom_toolchain_file:
  625. custom_toolchain_file = self.package_info.custom_toolchain_file
  626. custom_toolchain_file_path = pathlib.Path(custom_toolchain_file).absolute().resolve()
  627. if not custom_toolchain_file_path.exists():
  628. raise BuildError(f"Custom toolchain file specified does not exist: {custom_toolchain_file}\n"
  629. f"Path resolved: {custom_toolchain_file_path} ")
  630. print(f'Using custom toolchain file at {custom_toolchain_file_path}')
  631. cmake_generator_args.append( f'-DCMAKE_TOOLCHAIN_FILE="{custom_toolchain_file_path}"')
  632. cmake_module_path = ""
  633. paths_to_join = []
  634. if self.package_info.depends_on_packages:
  635. paths_to_join = []
  636. for package_name, package_hash, subfolder_name in self.package_info.depends_on_packages:
  637. package_download_location = self.base_temp_folder / package_name / subfolder_name
  638. paths_to_join.append(str(package_download_location.resolve()))
  639. cmake_module_path = ';'.join(paths_to_join).replace('\\', '/')
  640. if cmake_module_path:
  641. cmake_generate_cmd.extend([f"-DCMAKE_MODULE_PATH={cmake_module_path}"])
  642. cmake_generate_cmd.extend(cmake_generator_args)
  643. # make sure it always installs into a prefix (ie, not the system!)
  644. cmake_generate_cmd.extend([f"-DCMAKE_INSTALL_PREFIX={str(install_target_folder.resolve())}"])
  645. call_result = subprocess.run(subp_args(cmake_generate_cmd),
  646. shell=True,
  647. capture_output=False,
  648. cwd=str(self.build_folder.parent.resolve()))
  649. if call_result.returncode != 0:
  650. raise BuildError(f"Error generating project for platform {self.package_info.platform_name}")
  651. cmake_build_args = self.platform_config.get(f'cmake_build_args_{config.lower()}') or \
  652. self.platform_config.get('cmake_build_args') or \
  653. []
  654. if self.package_info.cmake_build_args_common:
  655. cmake_build_args = cmake_build_args + self.package_info.cmake_build_args_common
  656. validate_args(cmake_build_args)
  657. cmake_build_cmd = [self.cmake_command,
  658. '--build', str(self.build_folder.name),
  659. '--config', config]
  660. cmake_build_cmd.extend(cmake_build_args)
  661. call_result = subprocess.run(subp_args(cmake_build_cmd),
  662. shell=True,
  663. capture_output=False,
  664. cwd=str(self.build_folder.parent.resolve()))
  665. if call_result.returncode != 0:
  666. raise BuildError(f"Error building project for platform {self.package_info.platform_name}")
  667. cmake_install_cmd = [self.cmake_command,
  668. '--install', str(self.build_folder.name),
  669. '--config', config]
  670. call_result = subprocess.run(subp_args(cmake_install_cmd),
  671. shell=True,
  672. capture_output=False,
  673. cwd=str(self.build_folder.parent.resolve()))
  674. if call_result.returncode != 0:
  675. raise BuildError(f"Error installing project for platform {self.package_info.platform_name}")
  676. if cmake_install_filter:
  677. # If an install filter was specified, then perform a copy from the intermediate temp install folder
  678. # to the target package folder, applying the filter rules defined in the 'cmake_install_filter'
  679. # attribute.
  680. source_root_folder = str(install_target_folder.resolve())
  681. glob_results = glob.glob(f'{source_root_folder}/**', recursive=True)
  682. for glob_result in glob_results:
  683. if os.path.isdir(glob_result):
  684. continue
  685. source_relative = os.path.relpath(glob_result, source_root_folder)
  686. matched = False
  687. for pattern in cmake_install_filter:
  688. if fnmatch.fnmatch(source_relative, pattern):
  689. matched = True
  690. break
  691. if matched:
  692. target_path = self.build_install_folder / source_relative
  693. target_folder_path = target_path.parent
  694. create_folder(target_folder_path)
  695. shutil.copy2(glob_result, str(target_folder_path.resolve()), follow_symlinks=False)
  696. def create_custom_env(self):
  697. custom_env = os.environ.copy()
  698. custom_env['TARGET_INSTALL_ROOT'] = str(self.build_install_folder.resolve())
  699. custom_env['PACKAGE_ROOT'] = str(self.package_install_root.resolve())
  700. custom_env['TEMP_FOLDER'] = str(self.base_temp_folder.resolve())
  701. custom_env['PYTHON_BINARY'] = sys.executable
  702. if self.package_info.depends_on_packages:
  703. package_folder_list = []
  704. for package_name, _, subfoldername in self.package_info.depends_on_packages:
  705. package_folder_list.append(str( (self.base_temp_folder / package_name / subfoldername).resolve().absolute()))
  706. custom_env['DOWNLOADED_PACKAGE_FOLDERS'] = ';'.join(package_folder_list)
  707. return custom_env
  708. def build_and_install_custom(self):
  709. """
  710. Build and install from source using custom commands defined by 'custom_build_cmd' and 'custom_install_cmd'
  711. """
  712. # we add TARGET_INSTALL_ROOT, TEMP_FOLDER and DOWNLOADED_PACKAGE_FOLDERS to the environ for both
  713. # build and install, as they are useful to refer to from scripts.
  714. env_to_use = self.create_custom_env()
  715. custom_build_cmds = self.platform_config.get('custom_build_cmd', [])
  716. if custom_build_cmds:
  717. # Construct the custom build command to execute
  718. full_custom_build_cmd = shlex.join(custom_build_cmds).format(python=sys.executable)
  719. call_result = subprocess.run(full_custom_build_cmd,
  720. shell=True,
  721. capture_output=False,
  722. cwd=str(self.base_folder),
  723. env=env_to_use)
  724. if call_result.returncode != 0:
  725. raise BuildError(f"Error executing custom build command {full_custom_build_cmd}")
  726. custom_install_cmds = self.platform_config.get('custom_install_cmd', [])
  727. if custom_install_cmds:
  728. # Construct the custom install command to execute
  729. full_custom_install_cmd = shlex.join(custom_install_cmds).format(python=sys.executable)
  730. call_result = subprocess.run(full_custom_install_cmd,
  731. shell=True,
  732. capture_output=False,
  733. cwd=str(self.base_folder),
  734. env=env_to_use)
  735. if call_result.returncode != 0:
  736. raise BuildError(f"Error executing custom install command {full_custom_install_cmd}")
  737. # Allow libraries to define a list of files to include via a json script that stores folder paths and
  738. # individual files in the "Install_Paths" array
  739. custom_install_jsons = self.platform_config.get('custom_install_json', [])
  740. for custom_install_json_file in custom_install_jsons:
  741. custom_json_full_path = os.path.join(self.base_folder, custom_install_json_file)
  742. print(f"Running custom install json file {custom_json_full_path}")
  743. custom_json_full_path_file = open(custom_json_full_path)
  744. custom_install_json = json.loads(custom_json_full_path_file.read())
  745. if not custom_install_json:
  746. raise BuildError(f"Error loading custom install json file {custom_install_json_file}")
  747. source_subfolder = None
  748. if "Source_Subfolder" in custom_install_json:
  749. source_subfolder = custom_install_json["Source_Subfolder"]
  750. for install_path in custom_install_json["Install_Paths"]:
  751. install_src_path = install_path
  752. if source_subfolder is not None:
  753. install_src_path = os.path.join(source_subfolder, install_src_path)
  754. resolved_src_path = os.path.join(env_to_use['TEMP_FOLDER'], install_src_path)
  755. resolved_target_path = os.path.join(env_to_use['TARGET_INSTALL_ROOT'], install_path)
  756. if os.path.isdir(resolved_src_path):
  757. # Newer versions of Python support the parameter dirs_exist_ok=True,
  758. # but that's not available in earlier Python versions.
  759. # It's useful to treat it as an error if the target exists, because that means that something has
  760. # already touched that folder and there might be unexpected behavior copying an entire tree into it.
  761. print(f" Copying directory '{resolved_src_path}' to '{resolved_target_path}'")
  762. shutil.copytree(resolved_src_path, resolved_target_path)
  763. elif os.path.isfile(resolved_src_path):
  764. print(f" Copying file '{resolved_src_path}' to '{resolved_target_path}'")
  765. os.makedirs(os.path.dirname(resolved_target_path), exist_ok=True)
  766. shutil.copy2(resolved_src_path, resolved_target_path)
  767. else:
  768. raise BuildError(f"Error executing custom install json {custom_install_json_file}, found invalid source path {resolved_src_path}")
  769. def check_build_keys(self, keys_to_check):
  770. """
  771. Check a platform configuration for specific build keys
  772. """
  773. config_specific_build_keys = []
  774. for config in self.build_configs:
  775. for build_key in keys_to_check:
  776. config_specific_build_keys.append(f'{build_key}_{config.lower()}')
  777. for platform_config_key in self.platform_config.keys():
  778. if platform_config_key in keys_to_check:
  779. return True
  780. elif platform_config_key in config_specific_build_keys:
  781. return True
  782. return False
  783. def copy_file_or_directory(self, src: pathlib.Path, dst: pathlib.Path, ignore_patterns):
  784. """
  785. if @src is a directory, makes a deep copy of it into @dst. In this case @ignore_patterns is used.
  786. if @src is a file, copies it as @dst, and will create all intermediate directories in @dst as needed.
  787. """
  788. print(f"Source file: {src}, Destination file: {dst}")
  789. if src.is_dir():
  790. # Recursive deep copy. Will create all subdirectories as needed.
  791. def custom_copy(src, dst):
  792. #If the destination file exists, we'll leave it as is, because
  793. #it was generated already by the the build commands.
  794. if os.path.exists(dst):
  795. print(f"Destination file '{dst}' already exists. Skipping.")
  796. else:
  797. shutil.copy2(src, dst)
  798. shutil.copytree(src, dst, ignore=shutil.ignore_patterns(*ignore_patterns), copy_function=custom_copy, dirs_exist_ok=True)
  799. else:
  800. # If the destination directory doesn't exist, shutil.copy2 raises an exception.
  801. # Take care of this first.
  802. dst.parent.mkdir(parents=True, exist_ok=True)
  803. shutil.copy2(src,dst)
  804. def copy_extra_files(self):
  805. """
  806. Copies any extra files specified in the build config into the destination folder for packaging.
  807. """
  808. extra_files_to_copy = self.package_info.extra_files_to_copy
  809. if extra_files_to_copy:
  810. for params in extra_files_to_copy:
  811. source = params[0]
  812. dest = params[1]
  813. if len(params) < 3:
  814. ignore_patterns = ["",]
  815. else:
  816. ignore_patterns = params[2]
  817. self.copy_file_or_directory(
  818. self.base_folder / source,
  819. self.package_install_root / dest,
  820. ignore_patterns
  821. )
  822. def build_for_platform(self):
  823. """
  824. Build for the current platform (host+target)
  825. """
  826. has_cmake_arguments = self.check_build_keys(['cmake_generate_args', 'cmake_build_args'])
  827. has_custom_arguments = self.check_build_keys(['custom_build_cmd', 'custom_install_cmd'])
  828. if has_cmake_arguments and has_custom_arguments:
  829. raise BuildError("Bad build config file. You cannot have both cmake_* and custom_* platform build commands at the same time.")
  830. if has_cmake_arguments:
  831. self.build_and_install_cmake()
  832. elif has_custom_arguments:
  833. self.build_and_install_custom()
  834. else:
  835. raise BuildError("Bad build config file. Missing generate and build commands (cmake or custom)")
  836. def generate_package_info(self):
  837. """
  838. Generate the package file (PackageInfo.json)
  839. """
  840. self.package_info.write_package_info(self.package_install_root)
  841. def generate_cmake(self):
  842. """
  843. Generate the find*.cmake file for the library
  844. """
  845. if self.cmake_find_template is not None:
  846. template_file_content = self.cmake_find_template.read_text("UTF-8", "ignore")
  847. find_cmake_content = string.Template(template_file_content).substitute(self.cmake_template_env)
  848. elif self.cmake_find_source is not None:
  849. find_cmake_content = self.cmake_find_source.read_text("UTF-8", "ignore")
  850. target_cmake_find_script = self.package_install_root / self.package_info.cmake_find_target
  851. target_cmake_find_script.write_text(find_cmake_content)
  852. def assemble_from_prebuilt_source(self):
  853. assert self.prebuilt_source
  854. assert self.prebuilt_args
  855. # Optionally clean the target package folder first
  856. if self.clean_build:
  857. delete_folder(self.package_install_root)
  858. # Prepare the target package folder
  859. delete_folder(self.build_install_folder)
  860. create_folder(self.build_install_folder)
  861. prebuilt_source_path = (self.base_folder.resolve() / self.prebuilt_source).resolve()
  862. target_base_package_path = self.build_install_folder.resolve()
  863. # Loop through each of the prebuilt arguments (target/source glob pattern)
  864. for dest_path, glob_pattern in self.prebuilt_args.items():
  865. # Assemble the search pattern as a full path and keep track of the root of the search pattern so that
  866. # only the subpaths after the root of the search pattern will be copied to the target folder
  867. full_search_pattern = f"{str(prebuilt_source_path)}/{glob_pattern}"
  868. wildcard_index = full_search_pattern.find('*')
  869. source_base_folder_path = '' if wildcard_index < 0 else os.path.normpath(full_search_pattern[:wildcard_index])
  870. # Make sure the specified target folder exists
  871. target_base_folder_path = target_base_package_path / dest_path
  872. if target_base_folder_path.is_file():
  873. raise BuildError(f'Error: Target folder {target_base_folder_path} is a file')
  874. create_folder(target_base_folder_path)
  875. total_copied = 0
  876. # For each search pattern, run a glob
  877. glob_results = glob.glob(full_search_pattern, recursive=True)
  878. for glob_result in glob_results:
  879. if os.path.isdir(glob_result):
  880. continue
  881. source_relative = os.path.relpath(glob_result, source_base_folder_path)
  882. target_path = target_base_folder_path / source_relative
  883. target_folder_path = target_path.parent
  884. create_folder(target_folder_path)
  885. shutil.copy2(glob_result, str(target_folder_path.resolve()), follow_symlinks=False)
  886. total_copied += 1
  887. print(f"{total_copied} files copied to {target_base_folder_path}")
  888. pass
  889. def test_package(self):
  890. custom_test_cmd = self.platform_config.get('custom_test_cmd', [])
  891. if not custom_test_cmd:
  892. print(f"\n\nNo tests defined, skipping test phase.")
  893. return
  894. # Construct the custom build command to execute
  895. full_custom_test_cmd = shlex.join(custom_test_cmd).format(python=sys.executable)
  896. print(f"\n\nRunning custom test...")
  897. call_result = subprocess.run(full_custom_test_cmd,
  898. shell=True,
  899. capture_output=False,
  900. cwd=str(self.base_folder),
  901. env=self.create_custom_env())
  902. if call_result.returncode != 0:
  903. raise BuildError(f"Error executing custom test command {custom_test_cmd}")
  904. print(f"\n... Tests OK!")
  905. def execute(self):
  906. """
  907. Perform all the steps to build a folder for the 3rd party library for packaging
  908. """
  909. # Prepare the temp folder structure
  910. if self.prebuilt_source:
  911. self.assemble_from_prebuilt_source()
  912. else:
  913. self.prepare_temp_folders()
  914. # Sync Source
  915. self.sync_source()
  916. # Build the package
  917. self.build_for_platform()
  918. # Copy extra files specified in the build config
  919. self.copy_extra_files()
  920. # Generate the Find*.cmake file
  921. self.generate_cmake()
  922. self.test_package()
  923. # Generate the package info file
  924. self.generate_package_info()
  925. def prepare_build(platform_name, base_folder, build_folder, package_root_folder, cmake_command, build_config_file,
  926. clean, src_folder, skip_git):
  927. """
  928. Prepare a Build manager object based on parameters provided (possibly from command line)
  929. :param platform_name: The name of the target platform that the package is being for
  930. :param base_folder: The base folder where the build_config exists
  931. :param build_folder: The root folder to build into
  932. :param package_root_folder: The root of the package folder where the new package will be assembled
  933. :param cmake_command: The cmake executable command to use for cmake
  934. :param build_config_file: The build config file to open from the base_folder
  935. :param clean: Option to clean any existing build folder before proceeding
  936. :param src_folder: Option to manually specify the src folder
  937. :param skip_git: Option to skip all git commands, requires src_folder be supplied
  938. :return: The Build management object
  939. """
  940. base_folder_path = pathlib.Path(base_folder)
  941. build_folder_path = pathlib.Path(build_folder) if build_folder else base_folder_path / "temp"
  942. package_install_root = pathlib.Path(package_root_folder)
  943. src_folder_path = pathlib.Path(src_folder) if src_folder else build_folder_path / "src"
  944. if skip_git and src_folder is None:
  945. raise BuildError("Specified to skip git interactions but didn't supply a source code path")
  946. if src_folder is not None and not src_folder_path.is_dir():
  947. raise BuildError(f"Invalid path for 'git-path': {src_folder}")
  948. build_config_path = base_folder_path / build_config_file
  949. if not build_config_path.is_file():
  950. raise BuildError(f"Invalid build config path ({build_config_path.absolute()}). ")
  951. with build_config_path.open() as build_json_file:
  952. build_config = json.load(build_json_file)
  953. try:
  954. eligible_platforms = build_config["Platforms"][platform.system()]
  955. target_platform_config = eligible_platforms[platform_name]
  956. # Check if the target platform is an alias to another platform from the current eligible_platforms
  957. if isinstance(target_platform_config, str) and target_platform_config[0] == '@':
  958. target_platform_config = eligible_platforms[target_platform_config[1:]]
  959. except KeyError as e:
  960. raise BuildError(f"Invalid build config : {str(e)}")
  961. # Check if this is a prebuilt package to validate any additional required arguments
  962. prebuilt_source = target_platform_config.get('prebuilt_source') or build_config.get('prebuilt_source')
  963. if prebuilt_source:
  964. prebuilt_path = base_folder_path / prebuilt_source
  965. if not prebuilt_path.is_dir():
  966. raise BuildError(f"Invalid path given for 'prebuilt_source': {prebuilt_source}")
  967. prebuilt_args = target_platform_config.get('prebuilt_args')
  968. if not prebuilt_args:
  969. raise BuildError(f"Missing required 'prebuilt_args' argument for platform {platform_name}")
  970. else:
  971. prebuilt_args = None
  972. package_info = PackageInfo(build_config=build_config,
  973. target_platform_name=platform_name,
  974. target_platform_config=target_platform_config)
  975. cmake_find_template_path = None
  976. cmake_find_source_path = None
  977. if package_info.cmake_find_template is not None:
  978. # Validate the cmake find template
  979. if os.path.isabs(package_info.cmake_find_template):
  980. raise BuildError("Invalid 'cmake_find_template' entry in build config. Absolute paths are not allowed, must be relative to the package base folder.")
  981. cmake_find_template_path = base_folder_path / package_info.cmake_find_template
  982. if not cmake_find_template_path.is_file():
  983. raise BuildError("Invalid 'cmake_find_template' entry in build config")
  984. elif package_info.cmake_find_source is not None:
  985. # Validate the cmake find source
  986. if os.path.isabs(package_info.cmake_find_source):
  987. raise BuildError("Invalid 'cmake_find_source' entry in build config. Absolute paths are not allowed, must be relative to the package base folder.")
  988. cmake_find_source_path = base_folder_path / package_info.cmake_find_source
  989. if not cmake_find_source_path.is_file():
  990. raise BuildError("Invalid 'cmake_find_source' entry in build config")
  991. else:
  992. raise BuildError("Bad build config file. 'cmake_find_template' or 'cmake_find_template' must be specified.")
  993. return BuildInfo(package_info=package_info,
  994. platform_config=target_platform_config,
  995. base_folder=base_folder_path,
  996. build_folder=build_folder_path,
  997. package_install_root=package_install_root,
  998. cmake_command=cmake_command,
  999. clean_build=clean,
  1000. cmake_find_template=cmake_find_template_path,
  1001. cmake_find_source=cmake_find_source_path,
  1002. prebuilt_source=prebuilt_source,
  1003. prebuilt_args=prebuilt_args,
  1004. src_folder=src_folder_path,
  1005. skip_git=skip_git)
  1006. if __name__ == '__main__':
  1007. try:
  1008. parser = argparse.ArgumentParser(description="Tool to prepare a 3rd Party Folder for packaging for an open source project pulled from Git.",
  1009. formatter_class=argparse.RawDescriptionHelpFormatter,
  1010. epilog=SCHEMA_DESCRIPTION)
  1011. parser.add_argument('base_path',
  1012. help='The base path where the build configuration exists')
  1013. parser.add_argument('--platform-name',
  1014. help='The platform to build the package for.',
  1015. required=True)
  1016. parser.add_argument('--package-root',
  1017. help="The root path where to install the built packages to. This defaults to the {base_path}/temp. ",
  1018. required=False)
  1019. parser.add_argument('--cmake-path',
  1020. help='Path to where cmake is installed. Defaults to the system installed one.',
  1021. default='')
  1022. parser.add_argument('--build-config-file',
  1023. help=f"Filename of the build config file within the base_path. Defaults to '{DEFAULT_BUILD_CONFIG_FILENAME}'.",
  1024. default=DEFAULT_BUILD_CONFIG_FILENAME)
  1025. parser.add_argument('--clean',
  1026. help=f"Option to clean the build folder for a clean rebuild",
  1027. action="store_true")
  1028. parser.add_argument('--build-path',
  1029. help="Path to build the repository in. Defaults to {base_path}/temp.")
  1030. parser.add_argument('--source-path',
  1031. help='Path to a folder. Can be used to specify the git sync folder or provide an existing folder with source for the library.',
  1032. default=None)
  1033. parser.add_argument('--git-skip',
  1034. help='skips all git commands, requires source-path to be provided',
  1035. default=False)
  1036. parsed_args = parser.parse_args(sys.argv[1:])
  1037. # If package_root is not supplied, default to {base_path}/temp
  1038. resolved_package_root = parsed_args.package_root or f'{parsed_args.base_path}/temp'
  1039. cmake_path = validate_cmake(f"{parsed_args.cmake_path}/cmake" if parsed_args.cmake_path else "cmake")
  1040. # Prepare for the build
  1041. build_info = prepare_build(platform_name=parsed_args.platform_name,
  1042. base_folder=parsed_args.base_path,
  1043. build_folder=parsed_args.build_path,
  1044. package_root_folder=resolved_package_root,
  1045. cmake_command=cmake_path,
  1046. build_config_file=parsed_args.build_config_file,
  1047. clean=parsed_args.clean,
  1048. src_folder=parsed_args.source_path,
  1049. skip_git=parsed_args.git_skip)
  1050. # Execute the generation of the 3P folder for packaging
  1051. build_info.execute()
  1052. exit(0)
  1053. except BuildError as err:
  1054. print(err)
  1055. exit(1)