[PCK] Move directory to the end of file, write exported/saved PCK in place.

core/io/file_access_pack.cpp

@@ -256,31 +256,37 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
 
 	int64_t pck_start_pos = f->get_position() - 4;
 
+	// Read header.
 	uint32_t version = f->get_32();
 	uint32_t ver_major = f->get_32();
 	uint32_t ver_minor = f->get_32();
-	f->get_32(); // patch number, not used for validation.
+	uint32_t ver_patch = f->get_32(); // Not used for validation.
 
-	ERR_FAIL_COND_V_MSG(version != PACK_FORMAT_VERSION, false, vformat("Pack version unsupported: %d.", version));
-	ERR_FAIL_COND_V_MSG(ver_major > GODOT_VERSION_MAJOR || (ver_major == GODOT_VERSION_MAJOR && ver_minor > GODOT_VERSION_MINOR), false, vformat("Pack created with a newer version of the engine: %d.%d.", ver_major, ver_minor));
+	ERR_FAIL_COND_V_MSG(version != PACK_FORMAT_VERSION_V3 && version != PACK_FORMAT_VERSION_V2, false, vformat("Pack version unsupported: %d.", version));
+	ERR_FAIL_COND_V_MSG(ver_major > GODOT_VERSION_MAJOR || (ver_major == GODOT_VERSION_MAJOR && ver_minor > GODOT_VERSION_MINOR), false, vformat("Pack created with a newer version of the engine: %d.%d.%d.", ver_major, ver_minor, ver_patch));
 
 	uint32_t pack_flags = f->get_32();
-	uint64_t file_base = f->get_64();
-
 	bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
-	bool rel_filebase = (pack_flags & PACK_REL_FILEBASE);
+	bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3.
 
-	for (int i = 0; i < 16; i++) {
-		//reserved
-		f->get_32();
+	uint64_t file_base = f->get_64();
+	if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) {
+		file_base += pck_start_pos;
 	}
 
-	int file_count = f->get_32();
-
-	if (rel_filebase) {
-		file_base += pck_start_pos;
+	if (version == PACK_FORMAT_VERSION_V3) {
+		// V3: Read directory offset and skip reserved part of the header.
+		uint64_t dir_offset = f->get_64() + pck_start_pos;
+		f->seek(dir_offset);
+	} else if (version == PACK_FORMAT_VERSION_V2) {
+		// V2: Directory directly after the header.
+		for (int i = 0; i < 16; i++) {
+			f->get_32(); // Reserved.
+		}
 	}
 
+	// Read directory.
+	int file_count = f->get_32();
 	if (enc_directory) {
 		Ref<FileAccessEncrypted> fae;
 		fae.instantiate();
@@ -314,7 +320,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
 		if (flags & PACK_FILE_REMOVAL) { // The file was removed.
 			PackedData::get_singleton()->remove_path(path);
 		} else {
-			PackedData::get_singleton()->add_path(p_path, path, file_base + ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
+			PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
 		}
 	}
 

core/io/file_access_pack.h

@@ -38,8 +38,12 @@
 
 // Godot's packed file magic header ("GDPC" in ASCII).
 #define PACK_HEADER_MAGIC 0x43504447
+
+#define PACK_FORMAT_VERSION_V2 2
+#define PACK_FORMAT_VERSION_V3 3
+
 // The current packed file format version number.
-#define PACK_FORMAT_VERSION 2
+#define PACK_FORMAT_VERSION PACK_FORMAT_VERSION_V3
 
 enum PackFlags {
 	PACK_DIR_ENCRYPTED = 1 << 0,

core/io/pck_packer.cpp

@@ -95,14 +95,28 @@ Error PCKPacker::pck_start(const String &p_pck_path, int p_alignment, const Stri
 	file->store_32(GODOT_VERSION_MINOR);
 	file->store_32(GODOT_VERSION_PATCH);
 
-	uint32_t pack_flags = 0;
+	uint32_t pack_flags = PACK_REL_FILEBASE;
 	if (enc_dir) {
 		pack_flags |= PACK_DIR_ENCRYPTED;
 	}
 	file->store_32(pack_flags); // flags
 
+	file_base_ofs = file->get_position();
+	file->store_64(0); // Files base.
+
+	dir_base_ofs = file->get_position();
+	file->store_64(0); // Directory offset.
+
+	for (int i = 0; i < 16; i++) {
+		file->store_32(0); // Reserved.
+	}
+
+	file_base = file->get_position();
+	file->seek(file_base_ofs);
+	file->store_64(file_base); // Update files base.
+	file->seek(file_base);
+
 	files.clear();
-	ofs = 0;
 
 	return OK;
 }
@@ -114,7 +128,7 @@ Error PCKPacker::add_file_removal(const String &p_target_path) {
 	// Simplify path here and on every 'files' access so that paths that have extra '/'
 	// symbols or 'res://' in them still match the MD5 hash for the saved path.
 	pf.path = p_target_path.simplify_path().trim_prefix("res://");
-	pf.ofs = ofs;
+	pf.ofs = file->get_position();
 	pf.size = 0;
 	pf.removal = true;
 
@@ -138,7 +152,7 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa
 	// symbols or 'res://' in them still match the MD5 hash for the saved path.
 	pf.path = p_target_path.simplify_path().trim_prefix("res://");
 	pf.src_path = p_source_path;
-	pf.ofs = ofs;
+	pf.ofs = file->get_position();
 	pf.size = f->get_length();
 
 	Vector<uint8_t> data = FileAccess::get_file_as_bytes(p_source_path);
@@ -152,18 +166,29 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa
 	}
 	pf.encrypted = p_encrypt;
 
-	uint64_t _size = pf.size;
-	if (p_encrypt) { // Add encryption overhead.
-		if (_size % 16) { // Pad to encryption block size.
-			_size += 16 - (_size % 16);
-		}
-		_size += 16; // hash
-		_size += 8; // data size
-		_size += 16; // iv
+	Ref<FileAccess> ftmp = file;
+
+	Ref<FileAccessEncrypted> fae;
+	if (p_encrypt) {
+		fae.instantiate();
+		ERR_FAIL_COND_V(fae.is_null(), ERR_CANT_CREATE);
+
+		Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false);
+		ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE);
+		ftmp = fae;
 	}
 
-	int pad = _get_pad(alignment, ofs + _size);
-	ofs = ofs + _size + pad;
+	ftmp->store_buffer(data);
+
+	if (fae.is_valid()) {
+		ftmp.unref();
+		fae.unref();
+	}
+
+	int pad = _get_pad(alignment, file->get_position());
+	for (int j = 0; j < pad; j++) {
+		file->store_8(0);
+	}
 
 	files.push_back(pf);
 
@@ -173,14 +198,17 @@ Error PCKPacker::add_file(const String &p_target_path, const String &p_source_pa
 Error PCKPacker::flush(bool p_verbose) {
 	ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use.");
 
-	int64_t file_base_ofs = file->get_position();
-	file->store_64(0); // files base
-
-	for (int i = 0; i < 16; i++) {
-		file->store_32(0); // reserved
+	int dir_padding = _get_pad(alignment, file->get_position());
+	for (int i = 0; i < dir_padding; i++) {
+		file->store_8(0);
 	}
 
-	// write the index
+	// Write directory.
+	uint64_t dir_offset = file->get_position();
+	file->seek(dir_base_ofs);
+	file->store_64(dir_offset);
+	file->seek(dir_offset);
+
 	file->store_32(uint32_t(files.size()));
 
 	Ref<FileAccessEncrypted> fae;
@@ -196,7 +224,8 @@ Error PCKPacker::flush(bool p_verbose) {
 		fhead = fae;
 	}
 
-	for (int i = 0; i < files.size(); i++) {
+	const int file_num = files.size();
+	for (int i = 0; i < file_num; i++) {
 		CharString utf8_string = files[i].path.utf8();
 		int string_len = utf8_string.length();
 		int pad = _get_pad(4, string_len);
@@ -207,9 +236,9 @@ Error PCKPacker::flush(bool p_verbose) {
 			fhead->store_8(0);
 		}
 
-		fhead->store_64(files[i].ofs);
-		fhead->store_64(files[i].size); // pay attention here, this is where file is
-		fhead->store_buffer(files[i].md5.ptr(), 16); //also save md5 for file
+		fhead->store_64(files[i].ofs - file_base);
+		fhead->store_64(files[i].size);
+		fhead->store_buffer(files[i].md5.ptr(), 16);
 
 		uint32_t flags = 0;
 		if (files[i].encrypted) {
@@ -219,6 +248,10 @@ Error PCKPacker::flush(bool p_verbose) {
 			flags |= PACK_FILE_REMOVAL;
 		}
 		fhead->store_32(flags);
+
+		if (p_verbose) {
+			print_line(vformat("[%d/%d - %d%%] PCKPacker flush: %s -> %s", i, file_num, float(i) / file_num * 100, files[i].src_path, files[i].path));
+		}
 	}
 
 	if (fae.is_valid()) {
@@ -226,63 +259,12 @@ Error PCKPacker::flush(bool p_verbose) {
 		fae.unref();
 	}
 
-	int header_padding = _get_pad(alignment, file->get_position());
-	for (int i = 0; i < header_padding; i++) {
-		file->store_8(0);
-	}
-
-	uint64_t file_base = file->get_position();
-	file->seek(file_base_ofs);
-	file->store_64(file_base); // update files base
-	file->seek(file_base);
-
-	const uint32_t buf_max = 65536;
-	uint8_t *buf = memnew_arr(uint8_t, buf_max);
-
-	int count = 0;
-	for (int i = 0; i < files.size(); i++) {
-		if (files[i].removal) {
-			continue;
-		}
-
-		Ref<FileAccess> src = FileAccess::open(files[i].src_path, FileAccess::READ);
-		uint64_t to_write = files[i].size;
-
-		Ref<FileAccess> ftmp = file;
-		if (files[i].encrypted) {
-			fae.instantiate();
-			ERR_FAIL_COND_V(fae.is_null(), ERR_CANT_CREATE);
-
-			Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false);
-			ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE);
-			ftmp = fae;
-		}
-
-		while (to_write > 0) {
-			uint64_t read = src->get_buffer(buf, MIN(to_write, buf_max));
-			ftmp->store_buffer(buf, read);
-			to_write -= read;
-		}
-
-		if (fae.is_valid()) {
-			ftmp.unref();
-			fae.unref();
-		}
-
-		int pad = _get_pad(alignment, file->get_position());
-		for (int j = 0; j < pad; j++) {
-			file->store_8(0);
-		}
-
-		count += 1;
-		const int file_num = files.size();
-		if (p_verbose && (file_num > 0)) {
-			print_line(vformat("[%d/%d - %d%%] PCKPacker flush: %s -> %s", count, file_num, float(count) / file_num * 100, files[i].src_path, files[i].path));
-		}
-	}
-
 	file.unref();
-	memdelete_arr(buf);
-
 	return OK;
 }
+
+PCKPacker::~PCKPacker() {
+	if (file.is_valid()) {
+		flush();
+	}
+}

core/io/pck_packer.h

@@ -39,11 +39,14 @@ class PCKPacker : public RefCounted {
 
 	Ref<FileAccess> file;
 	int alignment = 0;
-	uint64_t ofs = 0;
 
 	Vector<uint8_t> key;
 	bool enc_dir = false;
 
+	uint64_t file_base = 0;
+	uint64_t file_base_ofs = 0;
+	uint64_t dir_base_ofs = 0;
+
 	static void _bind_methods();
 
 	struct File {
@@ -64,4 +67,5 @@ public:
 	Error flush(bool p_verbose = false);
 
 	PCKPacker() {}
+	~PCKPacker();
 };

doc/classes/PCKPacker.xml

@@ -31,7 +31,7 @@
 			<param index="1" name="source_path" type="String" />
 			<param index="2" name="encrypt" type="bool" default="false" />
 			<description>
-				Adds the [param source_path] file to the current PCK package at the [param target_path] internal path. The [code]res://[/code] prefix for [param target_path] is optional and stripped internally.
+				Adds the [param source_path] file to the current PCK package at the [param target_path] internal path. The [code]res://[/code] prefix for [param target_path] is optional and stripped internally. File content is immediately written to the PCK.
 			</description>
 		</method>
 		<method name="add_file_removal">
@@ -45,7 +45,8 @@
 			<return type="int" enum="Error" />
 			<param index="0" name="verbose" type="bool" default="false" />
 			<description>
-				Writes the files specified using all [method add_file] calls since the last flush. If [param verbose] is [code]true[/code], a list of files added will be printed to the console for easier debugging.
+				Writes the file directory and closes the PCK. If [param verbose] is [code]true[/code], a list of files added will be printed to the console for easier debugging.
+				[b]Note:[/b] [PCKPacker] will automatically flush when it's freed, which happens when it goes out of scope or when it gets assigned with [code]null[/code]. In C# the reference must be disposed after use, either with the [code]using[/code] statement or by calling the [code]Dispose[/code] method directly.
 			</description>
 		</method>
 		<method name="pck_start">

editor/export/editor_export_platform.cpp

@@ -305,10 +305,10 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
 		}
 
 		fae.instantiate();
-		ERR_FAIL_COND_V(fae.is_null(), ERR_SKIP);
+		ERR_FAIL_COND_V(fae.is_null(), ERR_FILE_CANT_OPEN);
 
 		Error err = fae->open_and_parse(ftmp, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv);
-		ERR_FAIL_COND_V(err != OK, ERR_SKIP);
+		ERR_FAIL_COND_V(err != OK, ERR_FILE_CANT_OPEN);
 		ftmp = fae;
 	}
 
@@ -320,6 +320,8 @@ Error EditorExportPlatform::_save_pack_file(void *p_userdata, const String &p_pa
 		fae.unref();
 	}
 
+	ERR_FAIL_COND_V(pd->f->get_position() - sd.ofs < (uint64_t)p_data.size(), ERR_FILE_CANT_WRITE);
+
 	int pad = _get_pad(PCK_PADDING, pd->f->get_position());
 	for (int i = 0; i < pad; i++) {
 		pd->f->store_8(0);
@@ -1906,53 +1908,19 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 	Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
 	da->make_dir_recursive(EditorPaths::get_singleton()->get_temp_dir());
 
-	String tmppath = EditorPaths::get_singleton()->get_temp_dir().path_join("packtmp");
-	Ref<FileAccess> ftmp = FileAccess::open(tmppath, FileAccess::WRITE);
-	if (ftmp.is_null()) {
-		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Cannot create file \"%s\"."), tmppath));
-		return ERR_CANT_CREATE;
-	}
-
-	PackData pd;
-	pd.ep = &ep;
-	pd.f = ftmp;
-	pd.so_files = p_so_files;
-
-	Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object);
-
-	// Close temp file.
-	pd.f.unref();
-	ftmp.unref();
-
-	if (err != OK) {
-		DirAccess::remove_file_or_error(tmppath);
-		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files."));
-		return err;
-	}
-
-	if (pd.file_ofs.is_empty()) {
-		DirAccess::remove_file_or_error(tmppath);
-		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export."));
-		return FAILED;
-	}
-
-	pd.file_ofs.sort(); //do sort, so we can do binary search later
-
 	Ref<FileAccess> f;
 	int64_t embed_pos = 0;
 	if (!p_embed) {
-		// Regular output to separate PCK file
+		// Regular output to separate PCK file.
 		f = FileAccess::open(p_path, FileAccess::WRITE);
 		if (f.is_null()) {
-			DirAccess::remove_file_or_error(tmppath);
 			add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for writing at path \"%s\"."), p_path));
 			return ERR_CANT_CREATE;
 		}
 	} else {
-		// Append to executable
+		// Append to executable.
 		f = FileAccess::open(p_path, FileAccess::READ_WRITE);
 		if (f.is_null()) {
-			DirAccess::remove_file_or_error(tmppath);
 			add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for reading-writing at path \"%s\"."), p_path));
 			return ERR_FILE_CANT_OPEN;
 		}
@@ -1973,32 +1941,68 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 
 	int64_t pck_start_pos = f->get_position();
 
+	// Write header.
 	f->store_32(PACK_HEADER_MAGIC);
 	f->store_32(PACK_FORMAT_VERSION);
 	f->store_32(GODOT_VERSION_MAJOR);
 	f->store_32(GODOT_VERSION_MINOR);
 	f->store_32(GODOT_VERSION_PATCH);
 
-	uint32_t pack_flags = 0;
+	uint32_t pack_flags = PACK_REL_FILEBASE;
 	bool enc_pck = p_preset->get_enc_pck();
 	bool enc_directory = p_preset->get_enc_directory();
 	if (enc_pck && enc_directory) {
 		pack_flags |= PACK_DIR_ENCRYPTED;
 	}
-	if (p_embed) {
-		pack_flags |= PACK_REL_FILEBASE;
-	}
-	f->store_32(pack_flags); // flags
+	f->store_32(pack_flags); // Flags.
 
 	uint64_t file_base_ofs = f->get_position();
-	f->store_64(0); // files base
+	f->store_64(0); // Files base.
+
+	uint64_t dir_base_ofs = f->get_position();
+	f->store_64(0); // Directory offset.
 
 	for (int i = 0; i < 16; i++) {
-		//reserved
-		f->store_32(0);
+		f->store_32(0); // Reserved.
+	}
+
+	uint64_t file_base = f->get_position();
+	f->seek(file_base_ofs);
+	f->store_64(file_base - pck_start_pos); // Update files base.
+	f->seek(file_base);
+
+	// Write files.
+	PackData pd;
+	pd.ep = &ep;
+	pd.f = f;
+	pd.so_files = p_so_files;
+
+	Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object);
+
+	if (err != OK) {
+		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files."));
+		return err;
 	}
 
-	f->store_32(pd.file_ofs.size()); //amount of files
+	if (pd.file_ofs.is_empty()) {
+		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export."));
+		return FAILED;
+	}
+
+	pd.file_ofs.sort(); // Do sort, so we can do binary search later (where ?).
+
+	int dir_padding = _get_pad(PCK_PADDING, f->get_position());
+	for (int i = 0; i < dir_padding; i++) {
+		f->store_8(0);
+	}
+
+	// Write directory.
+	uint64_t dir_offset = f->get_position();
+	f->seek(dir_base_ofs);
+	f->store_64(dir_offset - pck_start_pos);
+	f->seek(dir_offset);
+
+	f->store_32(pd.file_ofs.size());
 
 	Ref<FileAccessEncrypted> fae;
 	Ref<FileAccess> fhead = f;
@@ -2048,7 +2052,7 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 				for (int64_t j = 0; j < pd.file_ofs[i].md5.size(); j++) {
 					seed = ((seed << 5) + seed) ^ pd.file_ofs[i].md5[j];
 				}
-				seed = ((seed << 5) + seed) ^ pd.file_ofs[i].ofs;
+				seed = ((seed << 5) + seed) ^ (pd.file_ofs[i].ofs - file_base);
 				seed = ((seed << 5) + seed) ^ pd.file_ofs[i].size;
 			}
 
@@ -2078,9 +2082,9 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 			fhead->store_8(0);
 		}
 
-		fhead->store_64(pd.file_ofs[i].ofs);
-		fhead->store_64(pd.file_ofs[i].size); // pay attention here, this is where file is
-		fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16); //also save md5 for file
+		fhead->store_64(pd.file_ofs[i].ofs - file_base);
+		fhead->store_64(pd.file_ofs[i].size);
+		fhead->store_buffer(pd.file_ofs[i].md5.ptr(), 16);
 		uint32_t flags = 0;
 		if (pd.file_ofs[i].encrypted) {
 			flags |= PACK_FILE_ENCRYPTED;
@@ -2096,44 +2100,8 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 		fae.unref();
 	}
 
-	int header_padding = _get_pad(PCK_PADDING, f->get_position());
-	for (int i = 0; i < header_padding; i++) {
-		f->store_8(0);
-	}
-
-	uint64_t file_base = f->get_position();
-	uint64_t file_base_store = file_base;
-	if (pack_flags & PACK_REL_FILEBASE) {
-		file_base_store -= pck_start_pos;
-	}
-	f->seek(file_base_ofs);
-	f->store_64(file_base_store); // update files base
-	f->seek(file_base);
-
-	// Save the rest of the data.
-
-	ftmp = FileAccess::open(tmppath, FileAccess::READ);
-	if (ftmp.is_null()) {
-		DirAccess::remove_file_or_error(tmppath);
-		add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file to read from path \"%s\"."), tmppath));
-		return ERR_CANT_CREATE;
-	}
-
-	const int bufsize = 16384;
-	uint8_t buf[bufsize];
-
-	while (true) {
-		uint64_t got = ftmp->get_buffer(buf, bufsize);
-		if (got == 0) {
-			break;
-		}
-		f->store_buffer(buf, got);
-	}
-
-	ftmp.unref(); // Close temp file.
-
 	if (p_embed) {
-		// Ensure embedded data ends at a 64-bit multiple
+		// Ensure embedded data ends at a 64-bit multiple.
 		uint64_t embed_end = f->get_position() - embed_pos + 12;
 		uint64_t pad = embed_end % 8;
 		for (uint64_t i = 0; i < pad; i++) {
@@ -2150,8 +2118,6 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, b
 	}
 	f->close();
 
-	DirAccess::remove_file_or_error(tmppath);
-
 	return OK;
 }