android_deployment.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549
  1. #
  2. # Copyright (c) Contributors to the Open 3D Engine Project.
  3. # For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. #
  5. # SPDX-License-Identifier: Apache-2.0 OR MIT
  6. #
  7. #
  8. import datetime
  9. import logging
  10. import os
  11. import json
  12. import platform
  13. import subprocess
  14. import sys
  15. import time
  16. import pathlib
  17. from distutils.version import LooseVersion
  18. # Resolve the common python module
  19. ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
  20. if ROOT_DEV_PATH not in sys.path:
  21. sys.path.append(ROOT_DEV_PATH)
  22. from cmake.Tools import common
  23. from cmake.Tools.Platform.Android import android_support
  24. # The following is the list of known android external storage paths that we will attempt to verify on a device and
  25. # return the first one that is detected
  26. KNOWN_ANDROID_EXTERNAL_STORAGE_PATHS = [
  27. '/sdcard/',
  28. '/storage/emulated/0/',
  29. '/storage/emulated/legacy/',
  30. '/storage/sdcard0/',
  31. '/storage/self/primary/',
  32. ]
  33. ANDROID_TARGET_TIMESTAMP_FILENAME = 'deploy.timestamp'
  34. class AndroidDeployment(object):
  35. """
  36. Class to manage the deployment of game assets to an android device (Separately from the APK)
  37. """
  38. DEPLOY_APK_ONLY = 'APK'
  39. DEPLOY_ASSETS_ONLY = 'ASSETS'
  40. DEPLOY_BOTH = 'BOTH'
  41. def __init__(self, dev_root, build_dir, configuration, android_device_filter, clean_deploy, android_sdk_path, deployment_type, game_name=None, asset_mode=None, asset_type=None, embedded_assets=True, is_unit_test=False):
  42. """
  43. Initialize the Android Deployment Worker
  44. :param dev_root: The dev-root of the engine
  45. :param android_device_filter: An optional list of devices to filter on the connected devices to deploy to. If not supplied, deploy to all devices
  46. :param clean_deploy: Option to clean the target device's assets before deploying the game's assets from the host
  47. :param android_sdk_path: Path to the android SDK (to use the adb tool)
  48. :param deployment_type: The type of deployment (DEPLOY_APK_ONLY, DEPLOY_ASSETS_ONLY, or DEPLOY_BOTH)
  49. :param game_name: The name of the game whose assets are being deployed. None if is_test_project is True
  50. :param asset_mode: The asset mode of deployment (LOOSE, PAK, VFS). None if is_test_project is True
  51. :param asset_type: The asset type. None if is_test_project is True
  52. :param embedded_assets: Boolean to indicate if the assets are embedded in the APK or not
  53. :param is_unit_test: Boolean to indicate if this is a unit test deployment
  54. """
  55. self.dev_root = pathlib.Path(dev_root)
  56. self.build_dir = self.dev_root / build_dir
  57. self.configuration = configuration
  58. self.game_name = game_name
  59. self.asset_mode = asset_mode
  60. self.asset_type = asset_type
  61. self.clean_deploy = clean_deploy
  62. self.embedded_assets = embedded_assets
  63. self.deployment_type = deployment_type
  64. self.is_test_project = is_unit_test
  65. if not self.is_test_project:
  66. if embedded_assets:
  67. # If the assets are embedded, then warn that both APK and ASSETS will be deployed even if 'BOTH' is not specified
  68. if deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_ASSETS_ONLY):
  69. logging.warning(f"Deployment type of {deployment_type} set but the assets are embedded in the APK. Both the APK and the Assets will be deployed.")
  70. if asset_mode == 'PAK':
  71. self.local_asset_path = self.dev_root / 'Pak' / f'{game_name.lower()}_{asset_type}_paks'
  72. else:
  73. # Assets layout folder when assets are not included into APK is 'app/src/assets'
  74. self.local_asset_path = self.build_dir / 'app/src/assets'
  75. assert game_name is not None, f"'game_name' is required"
  76. self.game_name = game_name
  77. assert asset_mode is not None, f"'asset_mode' is required"
  78. self.asset_mode = asset_mode
  79. assert asset_type is not None, f"'asset_type' is required"
  80. self.asset_type = asset_type
  81. self.files_in_asset_path = list(self.local_asset_path.glob('**/*'))
  82. self.android_settings = AndroidDeployment.read_android_settings(self.dev_root, game_name)
  83. else:
  84. self.local_asset_path = None
  85. if asset_mode:
  86. logging.warning(f"'asset_mode' argument '{asset_mode}' ignored for unit test deployment.")
  87. if asset_type:
  88. logging.warning(f"'asset_type' argument '{asset_type}' ignored for unit test deployment.")
  89. self.files_in_asset_path = []
  90. self.apk_path = self.build_dir / 'app' / 'build' / 'outputs' / 'apk' / configuration / f'app-{configuration}.apk'
  91. self.android_device_filter = [android_device.strip() for android_device in android_device_filter.split(',')] if android_device_filter else []
  92. self.adb_path = AndroidDeployment.resolve_adb_tool(pathlib.Path(android_sdk_path))
  93. self.adb_started = False
  94. @staticmethod
  95. def read_android_settings(dev_root, game_name):
  96. """
  97. Read and parse the android_project.json file into a dictionary to process the specific attributes needed for the manifest template
  98. :param dev_root: The dev root we are working from
  99. :param game_name: Name of the game under the dev root
  100. :return: The android settings for the game project if any
  101. """
  102. game_folder = dev_root / game_name
  103. game_folder_project_properties_path = game_folder / 'Platform' / 'Android' / 'android_project.json'
  104. game_project_properties_content = game_folder_project_properties_path.resolve(strict=True)\
  105. .read_text(encoding=common.DEFAULT_TEXT_READ_ENCODING,
  106. errors=common.ENCODING_ERROR_HANDLINGS)
  107. # Extract the key attributes we need to process and build up our environment table
  108. game_project_json = json.loads(game_project_properties_content)
  109. android_settings = game_project_json.get('android_settings', {})
  110. return android_settings
  111. @staticmethod
  112. def resolve_adb_tool(android_sdk_path):
  113. """
  114. Resolve the location of the adb tool based on the input Android SDK Path
  115. :param android_sdk_path: The android SDK path to search for the adb tool
  116. :return: The absolute path to the adb tool
  117. """
  118. adb_target = 'adb.exe' if platform.system() == 'Windows' else 'adb'
  119. check_adb_target = android_sdk_path / 'platform-tools' / adb_target
  120. if not check_adb_target.exists():
  121. raise common.LmbrCmdError(f"Invalid Android SDK path '{str(android_sdk_path)}': Unable to locate '{adb_target}'.")
  122. return check_adb_target
  123. def get_android_project_settings(self, key_name, default_value):
  124. return self.android_settings.get(key_name, default_value)
  125. def adb_call(self, arg_list, device_id=None):
  126. """
  127. Wrapper to execute the adb command-line tool
  128. :param arg_list: Argument list to send to the tool
  129. :param device_id: Optional device id (from the 'get_target_android_devices' call) to invoke the call to.
  130. :return: The stdout result of the call
  131. """
  132. if isinstance(arg_list, str):
  133. arg_list = [arg_list]
  134. call_arguments = [str(self.adb_path.resolve())]
  135. if device_id:
  136. call_arguments.extend(['-s', device_id])
  137. call_arguments.extend(arg_list)
  138. logging.debug(f"adb command: {subprocess.list2cmdline(call_arguments)}")
  139. try:
  140. output = subprocess.check_output(subprocess.list2cmdline(call_arguments),
  141. shell=True,
  142. stderr=subprocess.PIPE).decode(common.DEFAULT_TEXT_READ_ENCODING,
  143. common.ENCODING_ERROR_HANDLINGS)
  144. logging.debug(f"adb output:\n{output}")
  145. return output
  146. except subprocess.CalledProcessError as err:
  147. std_out = err.stdout.decode(common.DEFAULT_TEXT_READ_ENCODING, common.ENCODING_ERROR_HANDLINGS)
  148. std_err = err.stderr.decode(common.DEFAULT_TEXT_READ_ENCODING, common.ENCODING_ERROR_HANDLINGS)
  149. logging.debug(f"adb returned non-zero.\noutput:\n{std_out}\nerror:\n{std_err}\n")
  150. raise common.LmbrCmdError(std_err)
  151. def adb_shell(self, command, device_id):
  152. """
  153. Special wrapper around calling "adb shell" which will invoke a shell command on the android device
  154. :param command: The shell command to invoke on the android device
  155. :param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
  156. :return: The stdout result of the call
  157. """
  158. shell_command = ['shell', command]
  159. return self.adb_call(shell_command, device_id=device_id)
  160. def adb_ls(self, path, device_id, args=None):
  161. """
  162. Request an 'ls' call on the android device
  163. :param path: The path to perform the 'ls' call on
  164. :param device_id: device id (from the 'get_target_android_devices' call) to invoke the shell call on
  165. :param args: Additional args to pass into the l'ls' call
  166. :return: Tuple of Boolean result of the call and the output of the call
  167. """
  168. error_messages = [
  169. 'No such file or directory',
  170. 'Permission denied'
  171. ]
  172. shell_command = ['ls']
  173. if args:
  174. shell_command.extend(args)
  175. shell_command.append(path)
  176. raw_output = self.adb_shell(command=' '.join(shell_command),
  177. device_id=device_id)
  178. if not raw_output:
  179. return False, None
  180. if raw_output is None or any([error for error in error_messages if error in raw_output]):
  181. status = False
  182. else:
  183. status = True
  184. return status, raw_output
  185. def get_target_android_devices(self):
  186. """
  187. Gets all of the connected android devices with adb, filtered by the set optional device filter
  188. :return: list of serial numbers of optionally filtered connected devices.
  189. """
  190. connected_devices = []
  191. # Call adb to get the device list and process the raw response
  192. raw_devices_output = self.adb_call("devices")
  193. if not raw_devices_output:
  194. raise common.LmbrCmdError("Error getting connected devices through adb")
  195. device_output_list = raw_devices_output.split(os.linesep)
  196. for device_output in device_output_list:
  197. if any(x in device_output for x in ['List', '*']):
  198. logging.debug(f"Skipping the following line as it has 'List', '*' or 'emulator' in it: {device_output}")
  199. continue
  200. device_serial = device_output.split()
  201. if device_serial:
  202. if 'unauthorized' in device_output.lower():
  203. logging.warning(f"Device {device_serial[0]} is not authorized for development access. Please reconnect the device and check for a confirmation dialog.")
  204. elif device_serial[0] in self.android_device_filter or not self.android_device_filter:
  205. connected_devices.append(device_serial[0])
  206. else:
  207. logging.debug(f"Skipping filtered out Device {device_serial[0]} .")
  208. if not connected_devices:
  209. raise common.LmbrCmdError("No connected android devices found")
  210. return connected_devices
  211. def check_known_android_paths(self, device_id):
  212. """
  213. Look for a known android path that is writeable and return the first one that is found
  214. :param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
  215. :return: The first available android path if found, None if not
  216. """
  217. for path in KNOWN_ANDROID_EXTERNAL_STORAGE_PATHS:
  218. logging.debug(f"Checking known path '{path}' on device '{device_id}'")
  219. # Test the path by performing an 'ls' call on it and checking if an error is returned from the result
  220. result, output = self.adb_ls(path=path,
  221. args=None,
  222. device_id=device_id)
  223. if result:
  224. return path[:-1]
  225. return None
  226. def detect_device_storage_path(self, device_id):
  227. """
  228. Uses the device's environment variable "EXTERNAL_STORAGE" to determine the correct
  229. path to public storage that has write permissions. If at any point does the env var
  230. validation fail, fallback to checking known possible paths to external storage.
  231. :param device_id:
  232. :return: The first available storage device
  233. """
  234. external_storage = self.adb_shell(command="set | grep EXTERNAL_STORAGE",
  235. device_id=device_id)
  236. if not external_storage:
  237. logging.debug(f"Unable to get 'EXTERNAL_STORAGE' environment from device '{device_id}'. Falling back to known android paths.")
  238. return self.check_known_android_paths(device_id)
  239. # Given the 'EXTERNAL_STORAGE' environment, parse out the value and validate it
  240. storage_path_key_value = external_storage.split('=')
  241. if len(storage_path_key_value) != 2:
  242. logging.debug(f"The value for 'EXTERNAL_STORAGE' environment from device '{device_id}' does not represent a valid key-value pair: {storage_path_key_value}. Falling back to known android paths")
  243. return self.check_known_android_paths(device_id)
  244. # Check the existence and permissions issue of the storage path
  245. storage_path = storage_path_key_value[1].strip()
  246. is_external_valid, _ = self.adb_ls(path=storage_path,
  247. device_id=device_id)
  248. if is_external_valid:
  249. return storage_path
  250. # The set external path has an issue, attempt to determine its real path through an adb shell call
  251. logging.debug(f"The path specified in EXTERNAL_STORAGE seems to have permission issues, attempting to resolve with realpath for device {device_id}.")
  252. real_path = self.adb_shell(command=f'realpath {storage_path}',
  253. device_id=device_id)
  254. if not real_path:
  255. logging.debug(f"Unable to determine the real path '{storage_path}' (from EXTERNAL_STORAGE) for {self.game_name} on device {device_id}. Falling back to known android paths")
  256. return self.check_known_android_paths(device_id)
  257. real_path = real_path.strip()
  258. is_external_valid, _ = self.adb_ls(path=real_path,
  259. device_id=device_id)
  260. if is_external_valid:
  261. return real_path
  262. logging.debug(f'Unable to validate the resolved EXTERNAL_STORAGE environment variable path for device {device_id}.')
  263. return self.check_known_android_paths(device_id)
  264. def get_device_file_timestamp(self, remote_file_path, device_id):
  265. """
  266. Get the integer timestamp value of a file from a given device.
  267. :param remote_file_path: The path to the timestamp file on the android device
  268. :param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
  269. :return: The time value if found, None if not
  270. """
  271. try:
  272. timestamp_string = self.adb_shell(command=f'cat {remote_file_path}',
  273. device_id=device_id).strip()
  274. except (common.LmbrCmdError, AttributeError):
  275. return None
  276. if not timestamp_string:
  277. return None
  278. for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f'):
  279. try:
  280. target_time = time.mktime(time.strptime(timestamp_string, fmt))
  281. break
  282. except ValueError:
  283. target_time = None
  284. return target_time
  285. def update_device_file_timestamp(self, relative_assets_path, device_id):
  286. """
  287. Update the device timestamp file on an android device to track files that need updating on pushes
  288. :param relative_assets_path: The relative path to the assets on the android device
  289. :param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
  290. """
  291. timestamp_str = str(datetime.datetime.now())
  292. logging.debug(f"Updating timestamp on device {device_id} to {timestamp_str}")
  293. local_timestamp_file_path = self.local_asset_path / ANDROID_TARGET_TIMESTAMP_FILENAME
  294. local_timestamp_file_path.write_text(timestamp_str)
  295. target_timestamp_file_path = f'{relative_assets_path}/{ANDROID_TARGET_TIMESTAMP_FILENAME}'
  296. self.adb_call(arg_list=['push', str(local_timestamp_file_path), target_timestamp_file_path],
  297. device_id=device_id)
  298. @staticmethod
  299. def should_copy_file(check_path, check_time):
  300. """
  301. Check if a source file should be copied, by checking if its timestamp is newer than the 'check_time'
  302. :param check_path: The path to the source file whose timestamp will be evaluated
  303. :param check_time: The baseline 'check_time' value to compare the source file timestamp against
  304. :return: True if the source file is newer than the baseline 'check_time', False if not
  305. """
  306. if not check_path.is_file():
  307. return False
  308. stat_src = check_path.stat()
  309. should_copy = stat_src.st_mtime >= check_time
  310. return should_copy
  311. def check_package_installed(self, package_name, target_device):
  312. """
  313. Checks if the package for the game is currently installed or not
  314. @param package_name: The name of the package to search for
  315. @param target_device: The target device to search for the package on
  316. @return: True if there an existing package on the device with the same package name, false if not
  317. """
  318. output_result = self.adb_call(['shell', 'cmd', 'package', 'list', 'packages', package_name],
  319. target_device)
  320. return output_result != ''
  321. def install_apk_to_device(self, target_device):
  322. """
  323. Install the APK to a target device
  324. @param target_device: The device id of the connected device to install to
  325. """
  326. if self.is_test_project:
  327. android_package_name = android_support.TEST_RUNNER_PACKAGE_NAME
  328. else:
  329. android_package_name = self.get_android_project_settings(key_name='package_name',
  330. default_value='org.o3de.sdk')
  331. if self.clean_deploy and self.check_package_installed(android_package_name, target_device):
  332. logging.info(f"Device '{target_device}': Uninstalling pre-existing APK for {self.game_name} ...")
  333. self.adb_call(arg_list=['uninstall', android_package_name],
  334. device_id=target_device)
  335. logging.info(f"Device '{target_device}': Installing APK for {self.game_name} ...")
  336. self.adb_call(arg_list=['install', '-t', '-r', str(self.apk_path.resolve())],
  337. device_id=target_device)
  338. def install_assets_to_device(self, detected_storage, target_device):
  339. """
  340. Install the assets for the game to a target device
  341. @param detected_storage: The detected storage path on the target device
  342. @param target_device: The ID of the target device
  343. """
  344. assert not self.is_test_project
  345. android_package_name = self.get_android_project_settings(key_name='package_name',
  346. default_value='org.o3de.sdk')
  347. relative_assets_path = f'Android/data/{android_package_name}/files'
  348. output_target = f'{detected_storage}/{relative_assets_path}'
  349. device_timestamp_file = f'{output_target}/{ANDROID_TARGET_TIMESTAMP_FILENAME}'
  350. # Track the current timestamp if possible to see if we can incrementally push files rather
  351. # than always pushing all files
  352. target_timestamp = self.get_device_file_timestamp(remote_file_path=device_timestamp_file,
  353. device_id=target_device)
  354. if self.clean_deploy:
  355. logging.info(f"Device '{target_device}': Cleaning target assets before deployment...")
  356. self.adb_shell(command=f'rm -rf {output_target}',
  357. device_id=target_device)
  358. logging.info(f"Device '{target_device}': Target cleaned.")
  359. # '/.' is necessary to avoid copying folder 'assets' to destination, but its content.
  360. assets_layout_src = f'{str(self.local_asset_path)}/.'
  361. assets_layout_dst = f'{output_target}'
  362. if self.clean_deploy or not target_timestamp:
  363. logging.info(f"Device '{target_device}': Pushing {len(self.files_in_asset_path)} files from {assets_layout_src} to device {assets_layout_dst} ...")
  364. try:
  365. self.adb_call(arg_list=['push', assets_layout_src, assets_layout_dst],
  366. device_id=target_device)
  367. except common.LmbrCmdError as err:
  368. # Something went wrong, clean up before leaving
  369. self.adb_shell(command=f'rm -rf {output_target}',
  370. device_id=target_device)
  371. raise err
  372. else:
  373. # If no clean was specified, individually inspect all files to see if it needs to be updated
  374. files_to_copy = []
  375. for asset_file in self.files_in_asset_path:
  376. # TODO: Check if the target exists in the destination as well?
  377. if AndroidDeployment.should_copy_file(asset_file, target_timestamp):
  378. files_to_copy.append(asset_file)
  379. if len(files_to_copy) > 0:
  380. logging.info(f"Copying {len(files_to_copy)} assets to device {target_device}")
  381. for src_path in files_to_copy:
  382. relative_path = os.path.relpath(str(src_path), str(self.local_asset_path)).replace('\\', '/')
  383. target_path = f"{output_target}/{relative_path}"
  384. self.adb_call(arg_list=['push', str(src_path), target_path],
  385. device_id=target_device)
  386. self.update_device_file_timestamp(relative_assets_path=output_target,
  387. device_id=target_device)
  388. def execute(self):
  389. """
  390. Execute the asset deployment
  391. """
  392. if self.is_test_project:
  393. if not self.apk_path.is_file():
  394. raise common.LmbrCmdError(f"Missing apk for {android_support.TEST_RUNNER_PROJECT} ({str(self.apk_path)}). Make sure it is built and is set as a signed APK.")
  395. else:
  396. if self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_BOTH):
  397. if not self.apk_path.is_file():
  398. raise common.LmbrCmdError(f"Missing apk for game {self.game_name} ({str(self.apk_path)}). Make sure it is built and is set as a signed APK.")
  399. if not self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_ASSETS_ONLY, AndroidDeployment.DEPLOY_BOTH):
  400. if not self.local_asset_path.is_dir():
  401. raise common.LmbrCmdError(f"Missing {self.asset_type} assets for game {self.game_name} .")
  402. try:
  403. logging.debug("Starting ADB Server")
  404. self.adb_call('start-server')
  405. self.adb_started = True
  406. # Get the list of target devices to deploy to
  407. target_devices = self.get_target_android_devices()
  408. if not target_devices:
  409. raise common.LmbrCmdError("No connected and eligible android devices found")
  410. for target_device in target_devices:
  411. detected_storage = self.detect_device_storage_path(target_device)
  412. if not detected_storage:
  413. logging.warning(f"Unable to resolve storage path for device '{target_device}'. Skipping.")
  414. continue
  415. if self.is_test_project:
  416. # If this is the unit test runner, then only install the APK, assets are not applicable
  417. self.install_apk_to_device(target_device=target_device)
  418. else:
  419. # Otherwise install the apk and assets based on the deployment type
  420. if self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_BOTH):
  421. self.install_apk_to_device(target_device=target_device)
  422. if not self.embedded_assets and self.deployment_type in (AndroidDeployment.DEPLOY_ASSETS_ONLY, AndroidDeployment.DEPLOY_BOTH):
  423. if self.deployment_type == AndroidDeployment.DEPLOY_ASSETS_ONLY:
  424. # If we are deploying assets only without an APK, make sure the APK is even installed first
  425. android_package_name = self.get_android_project_settings(key_name='package_name',
  426. default_value='org.o3de.sdk')
  427. if not self.check_package_installed(package_name=android_package_name,
  428. target_device=target_device):
  429. raise common.LmbrCmdError(f"Unable to locate APK for {self.game_name} on device '{target_device}'. Make sure it is installed "
  430. f"first before installing the assets.")
  431. self.install_assets_to_device(detected_storage=detected_storage,
  432. target_device=target_device)
  433. logging.info(f"{self.game_name} deployed to device {target_device}")
  434. finally:
  435. if self.adb_started:
  436. self.adb_call('kill-server')