using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; namespace OpenVIII { public sealed class ArchiveWorker : ArchiveBase { #region Fields public readonly ArchiveBase FsArchive; #endregion Fields #region Constructors /// /// Saves the active archive and file list. /// /// Memory.Archive /// If list generation is unneeded you can skip it by setting true private ArchiveWorker(Memory.Archive archive, bool skipList = false) { if (archive.IsDir) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}:: opening directory: {archive}"); IsDir = true; } else Memory.Log.WriteLine($"{nameof(ArchiveWorker)}:: opening archiveFile: {archive}"); Archive = archive; ParentPath = FindParentPath(archive); ArchiveBase tempArchive = null; if (ParentPath != null && ParentPath.Count > 0) foreach (var p in ParentPath) { if (tempArchive != null) { tempArchive = tempArchive.GetArchive(p); } else if (p.IsDir || p.IsFile) { tempArchive = ArchiveBase.Load(p); } } if (tempArchive != null) { tempArchive.GetArchive(archive, out var fi, out FsArchive, out var fl); ArchiveMap = new ArchiveMap(fi, fl, tempArchive.GetMaxSize(archive)); } if (!skipList) GetListOfFiles(); IsOpen = true; } private ArchiveWorker(Memory.Archive archive, StreamWithRangeValues fI, ArchiveBase fS, StreamWithRangeValues fL, bool skipList = false) { ArchiveMap = new ArchiveMap(fI, fL, fS.GetMaxSize(archive)); Archive = archive; FsArchive = fS; //FS = null; if (!skipList) GetListOfFiles(); IsOpen = true; } #endregion Constructors #region Methods /// /// Load Archive with out storing FS in byte[] works for archive that aren't compressed. /// /// Archive file archive. /// Stream containing the FI file /// Archive where the FS file is. /// Stream containing the FL file /// Skip generating list of files /// ArchiveWorker public static ArchiveBase Load(Memory.Archive path, StreamWithRangeValues fI, ArchiveBase fS, StreamWithRangeValues fL, bool skipList = false) { if (CacheTryGetValue(path, out var value)) { return value; } value = new ArchiveWorker(path, fI, fS, fL, skipList); if (!value.IsOpen) value = null; if (CacheTryAdd(path, value)) { } return value; } public static ArchiveBase Load(Memory.Archive path, bool skipList = false) { if (path.IsZZZ) return ArchiveZzz.Load(path, skipList); if (CacheTryGetValue(path, out var value)) { return value; } value = new ArchiveWorker(path, skipList); if (!value.IsOpen) value = null; if (CacheTryAdd(path, value)) { } return value; } public override ArchiveBase GetArchive(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) throw new FileNotFoundException("NO FILENAME"); if (ArchiveMap != null && ArchiveMap.Count > 1) FindFile(ref fileName); else if (IsDir) { if (FileList == null || FileList.Length == 0) ProduceFileLists(); fileName = FileList.FirstOrDefault(x => x.IndexOf(fileName, StringComparison.OrdinalIgnoreCase) >= 0); } else if (File.Exists(Archive.FL)) using (var fs = new FileStream(Archive.FL, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) FindFile(ref fileName, fs); return GetArchive((Memory.Archive)fileName); } public override ArchiveBase GetArchive(Memory.Archive archive) { if (archive == Memory.Archives.ZZZ_MAIN || archive == Memory.Archives.ZZZ_OTHER) { var zzz = archive.ZZZ; if (FindFile(ref zzz) <= -1 || string.IsNullOrWhiteSpace(zzz)) return null; if (File.Exists(zzz)) archive.SetFilename(zzz); return !CacheTryGetValue(archive, out var ab) ? ArchiveZzz.Load(zzz) : ab; } if (CacheTryGetValue(archive, out var value)) return value; GetArchive(archive, out var fI, out var fS, out StreamWithRangeValues fL); return fI == null || fS == null || fL == null || fI.Length == 0 || fL.Length == 0 ? null : new ArchiveWorker(archive, fI, fS, fL); } /// /// Get binary data /// /// filename you want /// if true store the data for later /// public override byte[] GetBinaryFile(string fileName, bool cache = false) { if (string.IsNullOrWhiteSpace(fileName)) throw new FileNotFoundException("NO FILENAME"); byte[] FileInTwoArchives() { //if (FS != null && FS.Length > 0) // return ArchiveMap.GetBinaryFile(filename, new MemoryStream(FS)); //else return FsArchive != null ? ArchiveMap.GetBinaryFile(fileName, FsArchive.GetStreamWithRangeValues(Archive.FS)) : null; } if (ArchiveMap != null && ArchiveMap.Count > 0) { if (!cache) return FileInTwoArchives(); var fileNameBackup = fileName; GetListOfFiles(); fileName = FileList.OrderBy(x => fileName.Length).ThenBy(x => fileNameBackup, StringComparer.OrdinalIgnoreCase).FirstOrDefault(x => x.IndexOf(fileNameBackup, StringComparison.OrdinalIgnoreCase) >= 0); if (string.IsNullOrWhiteSpace(fileName)) throw new NullReferenceException($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} fileName ({fileNameBackup}) not found!"); if (!LocalTryGetValue(fileName, out var value)) { var buffer = FileInTwoArchives(); if (LocalTryAdd(fileName, buffer)) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)}::{nameof(LocalTryAdd)} cached {fileName}"); } return buffer; } Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)}::{nameof(LocalTryGetValue)} read from cache {fileName}"); return value; } if (!IsDir) { var loc = -1; if (File.Exists(Archive.FL)) using (var fs = new FileStream(Archive.FL, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) loc = FindFile(ref fileName, fs); //File.OpenRead(archive.FL)); // read file list if (loc == -1) { Debug.WriteLine($"ArchiveWorker: NO SUCH FILE! :: {Archive.FL}"); //throw new Exception("ArchiveWorker: No such file!"); } else return GetBinaryFile(fileName, loc, cache); } else return GetBinaryFile(fileName, 0, cache); Debug.WriteLine($"ArchiveWorker: NO SUCH FILE! :: Searched {Archive} and could not find {fileName}"); return null; } public override Memory.Archive GetPath() => Archive; public override StreamWithRangeValues GetStreamWithRangeValues(string fileName) => GetStreamWithRangeValues(fileName, null, 0); public override string ToString() => $"{Archive} :: {Used}"; protected override int FindFile(ref string filename) { if (ArchiveMap != null && ArchiveMap.Count > 1) return base.FindFile(ref filename); if (IsDir) { var f = filename; filename = FileList.FirstOrDefault(x => x.IndexOf(f, StringComparison.OrdinalIgnoreCase) >= 0); return string.IsNullOrWhiteSpace(filename) ? -1 : 0; } using (var fs = new FileStream(Archive.FL, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) return FindFile(ref filename, fs); //File.OpenRead(archive.FL)); } /// /// Generate a file list from raw text file. /// /// protected override string[] ProduceFileLists() { string[] r; if (Archive == null) return null; if (IsDir) return Directory.GetFiles(Archive, "*", SearchOption.AllDirectories).OrderBy(x => x.Length) .ThenBy(x => x, StringComparer.OrdinalIgnoreCase).ToArray(); if ((r = base.ProduceFileLists()) != null) return r; if (!File.Exists(Archive.FL)) return null; using (var fs = new FileStream(Archive.FL, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) return ProduceFileLists(fs).OrderBy(x => x.Length).ThenBy(x => x, StringComparer.OrdinalIgnoreCase) .ToArray(); } /// /// Search file list for line filename is on. /// /// filename or archive to search for /// stream of text data to search in /// -1 on error or >=0 on success. private static int FindFile(ref string filename, Stream stream) { if (string.IsNullOrWhiteSpace(filename)) return -1; filename = filename.TrimEnd('\0'); using (TextReader tr = new StreamReader(stream)) { for (var i = 0; GetLine(tr, out var line); i++) { if (string.IsNullOrWhiteSpace(line)) { Debug.WriteLine("ArchiveWorker::File entry is null. Returning -1"); break; } if (line.IndexOf(filename, StringComparison.OrdinalIgnoreCase) < 0) continue; filename = line; //make sure the filename in dictionary is consistent. return i; } } Debug.WriteLine($"ArchiveWorker:: Filename {filename}, not found. returning -1"); return -1; } private static bool GetLine(TextReader tr, out string line) { line = tr.ReadLine(); if (string.IsNullOrWhiteSpace(line)) return false; line = line.TrimEnd('\0'); return true; } private static IEnumerable ProduceFileLists(Stream s) { using (var sr = new StreamReader(s, System.Text.Encoding.UTF8)) { var fl = new List(); while (!sr.EndOfStream) fl.Add(sr.ReadLine()?.Trim('\0', '\r', '\n')); return fl.ToArray(); } } private byte[] GetBinaryFile(string fileName, int loc, bool cache) { fileName = FileList.FirstOrDefault(x => x.IndexOf(fileName, StringComparison.OrdinalIgnoreCase) >= 0); if (LocalTryGetValue(fileName, out var b)) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} :: read from cache: {fileName}"); return b; } if (IsDir) { if (FileList == null || FileList.Length == 0) ProduceFileLists(); if (!string.IsNullOrWhiteSpace(fileName)) using (var br = new BinaryReader(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} :: reading: {fileName}"); var buffer = br.ReadBytes(checked((int)br.BaseStream.Length)); if (cache && LocalTryAdd(fileName, buffer)) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} :: cached: {fileName}"); } return buffer; } } //read index data var fi = GetFi(loc); //read binary data. var temp = GetCompressedData(fi, out var _); Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} :: extracting: {fileName}"); if (temp != null) switch (fi.CompressionType) { case CompressionType.None: break; case CompressionType.LZSS: LZSS.DecompressAllNew(temp, fi.UncompressedSize); break; case CompressionType.LZ4: temp = ArchiveMap.Lz4Uncompress(temp, fi.UncompressedSize); break; case CompressionType.LZSS_UnknownSize: LZSS.DecompressAllNew(temp, 0); break; case CompressionType.LZSS_LZSS: LZSS.DecompressAllNew(LZSS.DecompressAllNew(temp, fi.UncompressedSize), 0); break; default: throw new ArgumentOutOfRangeException(); } if (temp != null && cache && LocalTryAdd(fileName, temp)) { Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetBinaryFile)} :: cached: {fileName}"); } return temp; } /// /// GetCompressedData reads data from file directly. This isn't used right now I tried to add checks but they aren't tested yet. /// /// /// /// /// private byte[] GetCompressedData(FI fi, out int size, bool skipData = false) { byte[] temp = null; FileStream fs; using (var br = new BinaryReader(fs = new FileStream(Archive.FS, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { br.BaseStream.Seek(fi.Offset, SeekOrigin.Begin); var compressedSize = 0; var max = (int)(fs.Length - fs.Position); switch (fi.CompressionType) { case CompressionType.None: case CompressionType.LZSS_UnknownSize: compressedSize = fi.UncompressedSize; break; case CompressionType.LZSS: case CompressionType.LZSS_LZSS: if (fs.Length < 5) throw new InvalidDataException(); compressedSize = br.ReadInt32(); if (compressedSize > (max -= sizeof(int))) throw new InvalidDataException(); break; case CompressionType.LZ4: break; default: throw new ArgumentOutOfRangeException(); } // CompressionType = LZS then Compressed Size is defined. Else the Uncompressed size should work for size = compressedSize > 0 ? compressedSize : fi.UncompressedSize > max ? max : fi.UncompressedSize; if (size > max) { throw new InvalidDataException( $"{nameof(ArchiveWorker)}::{nameof(GetCompressedData)} Expected size ({size}) > ({max})"); } if (!skipData) temp = br.ReadBytes(size); } return temp; } private FI GetFi(int loc) { using (var br = new BinaryReader(new FileStream(Archive.FI, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))//File.OpenRead(archive.FI))) { br.BaseStream.Seek(loc * 12, SeekOrigin.Begin); return Extended.ByteArrayToClass(br.ReadBytes(12)); } } private StreamWithRangeValues GetStreamWithRangeValues(string fileName, FI inputFi, int size) { void msg() => Memory.Log.WriteLine($"{nameof(ArchiveWorker)}::{nameof(GetStreamWithRangeValues)} stream: {fileName}"); if (inputFi != null) { } if (Archive == null) return null; if (string.IsNullOrWhiteSpace(fileName)) throw new FileNotFoundException("NO FILENAME"); if (IsDir) { FindFile(ref fileName); msg(); var fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); if (inputFi == null) return new StreamWithRangeValues(fs, 0, fs.Length); return new StreamWithRangeValues(fs, inputFi.Offset, size, inputFi.CompressionType, inputFi.UncompressedSize); } Debug.Assert(inputFi == null); var parentFs = Archive.FS; if (ArchiveMap != null && ArchiveMap.Count > 1) { var fi = ArchiveMap.FindString(ref fileName, out size); msg(); if (FsArchive == null) return null; if (fi == null) return FsArchive.GetStreamWithRangeValues(fileName); var parentFi = FsArchive.ArchiveMap?.FindString(ref parentFs, out var _); return parentFi == null || parentFi.CompressionType == 0 || (FsArchive is ArchiveZzz) ? new StreamWithRangeValues(FsArchive.GetStreamWithRangeValues(parentFs), fi.Offset, size, fi.CompressionType, fi.UncompressedSize) : new StreamWithRangeValues(new MemoryStream(FsArchive.GetBinaryFile(parentFs, true), false), fi.Offset, size, fi.CompressionType, fi.UncompressedSize); } var loc = -1; if (File.Exists(Archive.FL)) using (var fs = new FileStream(Archive.FL, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) loc = FindFile(ref fileName, fs); //File.OpenRead(archive.FL)); msg(); // read file list if (loc == -1) { Debug.WriteLine($"ArchiveWorker: NO SUCH FILE! :: {Archive.FL}"); //throw new Exception("ArchiveWorker: No such file!"); } else { var fi = GetFi(loc); GetCompressedData(fi, out size, true); return new StreamWithRangeValues(new FileStream(parentFs, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), fi.Offset, size, fi.CompressionType, fi.UncompressedSize); } Debug.WriteLine($"ArchiveWorker: NO SUCH FILE! :: Searched {Archive} and could not find {fileName}"); return null; } #endregion Methods } }