Merge pull request #31706 from akien-mga/android-custom-build-improvements

Buildsystem and workflow improvements to the new Android custom builds

editor/editor_node.cpp

@@ -6373,13 +6373,13 @@ EditorNode::EditorNode() {
 	gui_base->add_child(custom_build_manage_templates);
 
 	install_android_build_template = memnew(ConfirmationDialog);
-	install_android_build_template->set_text(TTR("This will install the Android project for custom builds.\nNote that, in order to use it, it needs to be enabled per export preset."));
+	install_android_build_template->set_text(TTR("This will set up your project for custom Android builds by installing the source template to \"res://android/build\".\nYou can then apply modifications and build your own custom APK on export (adding modules, changing the AndroidManifest.xml, etc.).\nNote that in order to make custom builds instead of using pre-built APKs, the \"Use Custom Build\" option should be enabled in the Android export preset."));
 	install_android_build_template->get_ok()->set_text(TTR("Install"));
 	install_android_build_template->connect("confirmed", this, "_menu_confirm_current");
 	gui_base->add_child(install_android_build_template);
 
 	remove_android_build_template = memnew(ConfirmationDialog);
-	remove_android_build_template->set_text(TTR("Android build template is already installed and it won't be overwritten.\nRemove the \"build\" directory manually before attempting this operation again."));
+	remove_android_build_template->set_text(TTR("The Android build template is already installed in this project and it won't be overwritten.\nRemove the \"res://android/build\" directory manually before attempting this operation again."));
 	remove_android_build_template->get_ok()->set_text(TTR("Show in File Manager"));
 	remove_android_build_template->connect("confirmed", this, "_menu_option", varray(FILE_EXPLORE_ANDROID_BUILD_TEMPLATES));
 	gui_base->add_child(remove_android_build_template);

editor/export_template_manager.cpp

@@ -542,7 +542,6 @@ void ExportTemplateManager::_notification(int p_what) {
 		template_list_state->set_text(status);
 		if (errored) {
 			set_process(false);
-			;
 		}
 	}
 
@@ -555,25 +554,33 @@ void ExportTemplateManager::_notification(int p_what) {
 
 bool ExportTemplateManager::can_install_android_template() {
 
-	return FileAccess::exists(EditorSettings::get_singleton()->get_templates_dir().plus_file(VERSION_FULL_CONFIG).plus_file("android_source.zip"));
+	const String templates_dir = EditorSettings::get_singleton()->get_templates_dir().plus_file(VERSION_FULL_CONFIG);
+	return FileAccess::exists(templates_dir.plus_file("android_source.zip")) &&
+		   FileAccess::exists(templates_dir.plus_file("android_release.apk")) &&
+		   FileAccess::exists(templates_dir.plus_file("android_debug.apk"));
 }
 
 Error ExportTemplateManager::install_android_template() {
 
+	// To support custom Android builds, we install various things to the project's res://android folder.
+	// First is the Java source code and buildsystem from android_source.zip.
+	// Then we extract the Godot Android libraries from pre-build android_release.apk
+	// and android_debug.apk, to place them in the libs folder.
+
 	DirAccessRef da = DirAccess::open("res://");
 	ERR_FAIL_COND_V(!da, ERR_CANT_CREATE);
-	//make android dir (if it does not exist)
 
+	// Make res://android dir (if it does not exist).
 	da->make_dir("android");
 	{
-		//add an empty .gdignore file to avoid scan
+		// Add an empty .gdignore file to avoid scan.
 		FileAccessRef f = FileAccess::open("res://android/.gdignore", FileAccess::WRITE);
 		ERR_FAIL_COND_V(!f, ERR_CANT_CREATE);
 		f->store_line("");
 		f->close();
 	}
 	{
-		//add version, to ensure building won't work if template and Godot version don't match
+		// Add version, to ensure building won't work if template and Godot version don't match.
 		FileAccessRef f = FileAccess::open("res://android/.build_version", FileAccess::WRITE);
 		ERR_FAIL_COND_V(!f, ERR_CANT_CREATE);
 		f->store_line(VERSION_FULL_CONFIG);
@@ -583,7 +590,10 @@ Error ExportTemplateManager::install_android_template() {
 	Error err = da->make_dir_recursive("android/build");
 	ERR_FAIL_COND_V(err != OK, err);
 
-	String source_zip = EditorSettings::get_singleton()->get_templates_dir().plus_file(VERSION_FULL_CONFIG).plus_file("android_source.zip");
+	// Uncompress source template.
+
+	const String &templates_path = EditorSettings::get_singleton()->get_templates_dir().plus_file(VERSION_FULL_CONFIG);
+	const String &source_zip = templates_path.plus_file("android_source.zip");
 	ERR_FAIL_COND_V(!FileAccess::exists(source_zip), ERR_CANT_OPEN);
 
 	FileAccess *src_f = NULL;
@@ -593,37 +603,33 @@ Error ExportTemplateManager::install_android_template() {
 	ERR_FAIL_COND_V_MSG(!pkg, ERR_CANT_OPEN, "Android sources not in ZIP format.");
 
 	int ret = unzGoToFirstFile(pkg);
-
 	int total_files = 0;
-	//count files
+	// Count files to unzip.
 	while (ret == UNZ_OK) {
 		total_files++;
 		ret = unzGoToNextFile(pkg);
 	}
-
 	ret = unzGoToFirstFile(pkg);
-	//decompress files
-	ProgressDialog::get_singleton()->add_task("uncompress", TTR("Uncompressing Android Build Sources"), total_files);
 
-	Set<String> dirs_tested;
+	ProgressDialog::get_singleton()->add_task("uncompress_src", TTR("Uncompressing Android Build Sources"), total_files);
 
+	Set<String> dirs_tested;
 	int idx = 0;
 	while (ret == UNZ_OK) {
 
-		//get filename
+		// Get file path.
 		unz_file_info info;
-		char fname[16384];
-		ret = unzGetCurrentFileInfo(pkg, &info, fname, 16384, NULL, 0, NULL, 0);
-
-		String name = fname;
+		char fpath[16384];
+		ret = unzGetCurrentFileInfo(pkg, &info, fpath, 16384, NULL, 0, NULL, 0);
 
-		String base_dir = name.get_base_dir();
+		String path = fpath;
+		String base_dir = path.get_base_dir();
 
-		if (!name.ends_with("/")) {
+		if (!path.ends_with("/")) {
 			Vector<uint8_t> data;
 			data.resize(info.uncompressed_size);
 
-			//read
+			// Read.
 			unzOpenCurrentFile(pkg);
 			unzReadCurrentFile(pkg, data.ptrw(), data.size());
 			unzCloseCurrentFile(pkg);
@@ -633,7 +639,7 @@ Error ExportTemplateManager::install_android_template() {
 				dirs_tested.insert(base_dir);
 			}
 
-			String to_write = String("res://android/build").plus_file(name);
+			String to_write = String("res://android/build").plus_file(path);
 			FileAccess *f = FileAccess::open(to_write, FileAccess::WRITE);
 			if (f) {
 				f->store_buffer(data.ptr(), data.size());
@@ -646,13 +652,96 @@ Error ExportTemplateManager::install_android_template() {
 			}
 		}
 
-		ProgressDialog::get_singleton()->task_step("uncompress", name, idx);
+		ProgressDialog::get_singleton()->task_step("uncompress_src", path, idx);
+
+		idx++;
+		ret = unzGoToNextFile(pkg);
+	}
+
+	ProgressDialog::get_singleton()->end_task("uncompress_src");
+	unzClose(pkg);
+
+	// Extract libs from pre-built APKs.
+	err = _extract_libs_from_apk("release");
+	ERR_FAIL_COND_V_MSG(err != OK, err, "Can't extract Android libs from android_release.apk.");
+	err = _extract_libs_from_apk("debug");
+	ERR_FAIL_COND_V_MSG(err != OK, err, "Can't extract Android libs from android_debug.apk.");
+
+	return OK;
+}
+
+Error ExportTemplateManager::_extract_libs_from_apk(const String &p_target_name) {
+
+	const String &templates_path = EditorSettings::get_singleton()->get_templates_dir().plus_file(VERSION_FULL_CONFIG);
+	const String &apk_file = templates_path.plus_file("android_" + p_target_name + ".apk");
+	ERR_FAIL_COND_V(!FileAccess::exists(apk_file), ERR_CANT_OPEN);
+
+	FileAccess *src_f = NULL;
+	zlib_filefunc_def io = zipio_create_io_from_file(&src_f);
+
+	unzFile pkg = unzOpen2(apk_file.utf8().get_data(), &io);
+	ERR_FAIL_COND_V_MSG(!pkg, ERR_CANT_OPEN, "Android APK can't be extracted.");
+
+	DirAccessRef da = DirAccess::open("res://");
+	ERR_FAIL_COND_V(!da, ERR_CANT_CREATE);
+
+	// 8 steps because 4 arches, 2 libs per arch.
+	ProgressDialog::get_singleton()->add_task("extract_libs_from_apk", TTR("Extracting Android Libraries From APKs"), 8);
+
+	int ret = unzGoToFirstFile(pkg);
+	Set<String> dirs_tested;
+	int idx = 0;
+	while (ret == UNZ_OK) {
+		// Get file path.
+		unz_file_info info;
+		char fpath[16384];
+		ret = unzGetCurrentFileInfo(pkg, &info, fpath, 16384, NULL, 0, NULL, 0);
+
+		String path = fpath;
+		String base_dir = path.get_base_dir();
+		String file = path.get_file();
+
+		if (!base_dir.begins_with("lib") || path.ends_with("/")) {
+			ret = unzGoToNextFile(pkg);
+			continue;
+		}
+
+		Vector<uint8_t> data;
+		data.resize(info.uncompressed_size);
+
+		// Read.
+		unzOpenCurrentFile(pkg);
+		unzReadCurrentFile(pkg, data.ptrw(), data.size());
+		unzCloseCurrentFile(pkg);
+
+		// We have a "lib" folder in the APK, but it should be "libs/{release,debug}" in the source dir.
+		String target_base_dir = base_dir.replace_first("lib", String("libs").plus_file(p_target_name));
+
+		if (!dirs_tested.has(base_dir)) {
+			da->make_dir_recursive(String("android/build").plus_file(target_base_dir));
+			dirs_tested.insert(base_dir);
+		}
+
+		String to_write = String("res://android/build").plus_file(target_base_dir.plus_file(path.get_file()));
+		FileAccess *f = FileAccess::open(to_write, FileAccess::WRITE);
+		if (f) {
+			f->store_buffer(data.ptr(), data.size());
+			memdelete(f);
+#ifndef WINDOWS_ENABLED
+			// We can't retrieve Unix permissions from the APK it seems, so simply set 0755 as should be.
+			FileAccess::set_unix_permissions(to_write, 0755);
+#endif
+		} else {
+			ERR_PRINTS("Can't uncompress file: " + to_write);
+		}
+
+		ProgressDialog::get_singleton()->task_step("extract_libs_from_apk", path, idx);
 
 		idx++;
 		ret = unzGoToNextFile(pkg);
 	}
 
-	ProgressDialog::get_singleton()->end_task("uncompress");
+	ProgressDialog::get_singleton()->end_task("extract_libs_from_apk");
 	unzClose(pkg);
 
 	return OK;

editor/export_template_manager.h

@@ -72,6 +72,8 @@ class ExportTemplateManager : public ConfirmationDialog {
 	virtual void ok_pressed();
 	bool _install_from_file(const String &p_file, bool p_use_progress = true);
 
+	Error _extract_libs_from_apk(const String &p_target_name);
+
 	void _http_download_mirror_completed(int p_status, int p_code, const PoolStringArray &headers, const PoolByteArray &p_data);
 	void _http_download_templates_completed(int p_status, int p_code, const PoolStringArray &headers, const PoolByteArray &p_data);
 

platform/android/SCsub

@@ -55,3 +55,25 @@ if lib_arch_dir != '':
 
     stl_lib_path = str(env['ANDROID_NDK_ROOT']) + '/sources/cxx-stl/llvm-libc++/libs/' + lib_arch_dir + '/libc++_shared.so'
     env_android.Command(out_dir + '/libc++_shared.so', stl_lib_path, Copy("$TARGET", "$SOURCE"))
+
+# Zip android/java folder for the source export template.
+print("Archiving platform/android/java as bin/android_source.zip...")
+import os
+import zipfile
+# Change dir to avoid have zipped paths start from the android/java folder.
+olddir = os.getcwd()
+os.chdir(Dir('#platform/android/java').abspath)
+bindir = Dir('#bin').abspath
+# Make 'bin' dir if missing, can happen on fresh clone.
+if not os.path.exists(bindir):
+    os.makedirs(bindir)
+zipf = zipfile.ZipFile(os.path.join(bindir, 'android_source.zip'), 'w', zipfile.ZIP_DEFLATED)
+exclude_dirs = ['.gradle', 'build', 'libs', 'patches']
+for root, dirs, files in os.walk('.', topdown=True):
+    # Change 'dirs' in place to exclude folders we don't want.
+    # https://stackoverflow.com/a/19859907
+    dirs[:] = [d for d in dirs if d not in exclude_dirs]
+    for f in files:
+        zipf.write(os.path.join(root, f))
+zipf.close()
+os.chdir(olddir)

platform/android/export/export.cpp

@@ -1610,19 +1610,16 @@ public:
 				valid = false;
 			} else {
 				Error errn;
-				DirAccess *da = DirAccess::open(sdk_path.plus_file("tools"), &errn);
+				DirAccessRef da = DirAccess::open(sdk_path.plus_file("tools"), &errn);
 				if (errn != OK) {
 					err += TTR("Invalid Android SDK path for custom build in Editor Settings.") + "\n";
 					valid = false;
 				}
-				if (da) {
-					memdelete(da);
-				}
 			}
 
 			if (!FileAccess::exists("res://android/build/build.gradle")) {
 
-				err += TTR("Android project is not installed for compiling. Install from Editor menu.") + "\n";
+				err += TTR("Android build template not installed in the project. Install it from the Project menu.") + "\n";
 				valid = false;
 			}
 		}
@@ -2513,7 +2510,7 @@ void register_android_exporter() {
 	EDITOR_DEF("export/android/debug_keystore_pass", "android");
 	EDITOR_DEF("export/android/force_system_user", false);
 	EDITOR_DEF("export/android/custom_build_sdk_path", "");
-	EditorSettings::get_singleton()->add_property_hint(PropertyInfo(Variant::STRING, "export/android/custom_build_sdk_path", PROPERTY_HINT_GLOBAL_DIR, "*.keystore"));
+	EditorSettings::get_singleton()->add_property_hint(PropertyInfo(Variant::STRING, "export/android/custom_build_sdk_path", PROPERTY_HINT_GLOBAL_DIR));
 
 	EDITOR_DEF("export/android/timestamping_authority_url", "");
 	EDITOR_DEF("export/android/shutdown_adb_on_exit", true);