/******************************************************************************/ #include "stdafx.h" namespace EE{ /****************************************************************************** 'File._cipher_offset' is needed because of: -'Pak._cipher_per_file' can be either false or true -'File.limit/unlimit' is used for 'DecompressRaw' and 'ChunkReader' which will break if "Pak._cipher_per_file==true" -Android can have assets located within the APK with an offset (we can get a stdio file descriptor for APK and a custom file inside it at a certain offset in that file) which will break if "Pak._cipher_per_file==false" "Pak._cipher_per_file==true" is recommended because it improves Steam patching system: making updates will require smaller uploads/downloads, because Steam reuses same 1MB chunks: https://partner.steamgames.com/documentation/steampipe#Building_Efficient_Depots Because of buffering, the actual position (file descriptor) may be different than File.pos. Because in FILE_STD_READ more data is read, and placed into the buffer (descriptor being ahead of 'File.pos') and in FILE_STD_WRITE data is buffered, and written once buffer is full ('File.pos' being ahead of descriptor) See 'File.posFile' /****************************************************************************** BUF_SIZE was chosen based on speed tests using following code: Str file="d:/test.dat"; if(MOBILE)file=SystemPath(SP_APP_DATA).tailSlash(true)+"test.dat"; long size=(1024<<20); int mul=(MOBILE ? 1 : 4); File f; f.write(file); REP(mul)f.put(null, size); size*=mul; extern int BUF_SIZE; LogDel(); ProcPriority(2); LogN(S+"file size:"+size); for(BUF_SIZE=4096; BUF_SIZE<=(16<<20); BUF_SIZE*=2) { f.del(); f.read(file); Str s=S+"BUF_SIZE:"+BUF_SIZE; byte temp[4096]; flt t=Time.curTime(); f.pos(0); for(; !f.end(); )f.getByte(); s+=S+", 1:"+(Time.curTime()-t); t=Time.curTime(); f.pos(0); for(; !f.end(); )f.getUInt(); s+=S+", 4:"+(Time.curTime()-t); t=Time.curTime(); f.pos(0); for(; !f.end(); )f>>temp ; s+=S+", 4k:"+(Time.curTime()-t); LogN(s); } Results on Asus Zenbook Prime, 15" 512GB SSD Win Laptop: file size:4294967296 BUF_SIZE:4096, 1:56.978, 4:16.192, 4k:2.755 BUF_SIZE:8192, 1:55.714, 4:15.340, 4k:2.754 BUF_SIZE:16384, 1:55.269, 4:14.835, 4k:1.635 BUF_SIZE:32768, 1:55.097, 4:14.684, 4k:1.404 BUF_SIZE:65536, 1:54.998, 4:14.600, 4k:1.337 <- BEST BUF_SIZE:131072, 1:54.920, 4:14.636, 4k:1.416 BUF_SIZE:262144, 1:54.976, 4:14.791, 4k:1.363 BUF_SIZE:524288, 1:54.506, 4:14.718, 4k:1.354 BUF_SIZE:1048576, 1:54.886, 4:14.592, 4k:1.361 BUF_SIZE:2097152, 1:55.293, 4:14.513, 4k:1.364 BUF_SIZE:4194304, 1:55.138, 4:14.704, 4k:1.523 BUF_SIZE:8388608, 1:55.132, 4:14.898, 4k:1.715 Results on MacBook Air 13" Mid 2013 128 GB SSD: file size:1073741824 BUF_SIZE:4096, 1:22.485, 4:6.513, 4k:0.461 BUF_SIZE:8192, 1:21.548, 4:6.062, 4k:0.461 BUF_SIZE:16384, 1:21.118, 4:6.077, 4k:0.329 BUF_SIZE:32768, 1:21.408, 4:5.797, 4k:0.340 BUF_SIZE:65536, 1:21.656, 4:5.807, 4k:0.275 <- BEST BUF_SIZE:131072, 1:21.296, 4:5.955, 4k:0.331 BUF_SIZE:262144, 1:21.188, 4:5.991, 4k:0.325 BUF_SIZE:524288, 1:20.682, 4:6.036, 4k:0.321 BUF_SIZE:1048576, 1:21.438, 4:6.011, 4k:0.342 BUF_SIZE:2097152, 1:21.487, 4:5.918, 4k:0.368 BUF_SIZE:4194304, 1:21.127, 4:5.932, 4k:0.414 BUF_SIZE:8388608, 1:20.767, 4:6.236, 4k:0.428 BUF_SIZE:16777216, 1:20.651, 4:5.961, 4k:0.425 Results on iPad Mini 2: file size:1073741824 BUF_SIZE:4096, 1:34.880, 4:11.120, 4k:6.909 BUF_SIZE:8192, 1:34.407, 4:10.864, 4k:6.891 BUF_SIZE:16384, 1:34.199, 4:10.639, 4k:6.811 BUF_SIZE:32768, 1:34.171, 4:10.387, 4k:6.737 BUF_SIZE:65536, 1:34.142, 4:10.139, 4k:6.747 BUF_SIZE:131072, 1:34.137, 4:9.875, 4k:6.739 BUF_SIZE:262144, 1:34.117, 4:9.860, 4k:6.738 BUF_SIZE:524288, 1:34.292, 4:10.006, 4k:6.507 BUF_SIZE:1048576, 1:34.184, 4:10.021, 4k:5.950 BUF_SIZE:2097152, 1:36.769, 4:12.563, 4k:5.979 BUF_SIZE:4194304, 1:38.040, 4:13.808, 4k:5.956 BUF_SIZE:8388608, 1:38.799, 4:14.405, 4k:5.961 BUF_SIZE:16777216, 1:38.953, 4:14.765, 4k:5.981 /******************************************************************************/ #if DESKTOP #define BUF_SIZE (1<<16) // 64 KB #else #define BUF_SIZE (1<<15) // 32 KB (for Mobile devices use smaller buffers because they usually have less RAM) #endif #define TEMP_BUF_SIZE (1<<17) // 128 KB #define ALLOW_REFLUSH 1 // if allow secondary flush with remaining data when first failed /******************************************************************************/ #if WINDOWS static INLINE Long Seek(Int handle, Long offset, Int mode) {return _lseeki64(handle, offset, mode);} #elif APPLE static INLINE Long Seek(Int handle, Long offset, Int mode) {return lseek (handle, offset, mode);} // on Apple 'lseek' is already 64-bit #elif LINUX || ANDROID || WEB static INLINE Long Seek(Int handle, Long offset, Int mode) {return lseek64 (handle, offset, mode);} #endif static INLINE Int Write(Int handle, CPtr data, Int size) {return PLATFORM(_write, write)(handle, data, size);} static INLINE Int Read (Int handle, Ptr data, Int size) {return PLATFORM(_read , read )(handle, data, size);} /******************************************************************************/ // !! if adding any members here then adjust 'copyToAndDiscard', 'writeMemFixed' !! void File::zeroNoBuf() { _type=FILE_NONE; _writable=false; _ok=true; _path=FILE_PATH(0); _buf_pos=_buf_len=_cipher_offset=_handle=0; _offset=_pos=_size=0; _pak=null; _cipher=null; _mem=null; #if ANDROID _aasset=null; #endif } void File::zero() { zeroNoBuf(); _buf_size=0; _buf=null; } File::File( ) {zero();} File::File(C Str &name , Cipher *cipher) : File() {read (name , cipher);} File::File(C UID &id , Cipher *cipher) : File() {read (id , cipher);} File::File(C Str &name, C Pak &pak ) : File() {read (name, pak );} File::File(C PakFile &file, C Pak &pak ) : File() {read (file, pak );} File::File( CPtr data, Int size, Cipher *cipher) : File() {readMem(data, size, cipher);} void File::delBuf( ) {Free(_buf); _buf_size=0;} Bool File::setBuf(Int size) {if(size>_buf_size){delBuf(); if(_buf=Alloc(size))_buf_size=size;} return !size || _buf!=null;} File& File::del() { close(); delBuf(); // !! delete buffer after closing, because it may be used by 'flush' !! #if !FILE_MEMB_UNION _memb.del(); #endif return T; } void File::close() { switch(_type) { case FILE_STD_READ : case FILE_STD_WRITE: { flush(); // remember that this 'flush' may fail, and '_buf_len' may remain >0, however it will be cleared in 'zeroNoBuf' ::PLATFORM(_close, close)(_handle); #if WEB if(_writable)FlushIO(); // check for '_writable' instead of 'FILE_STD_WRITE' because File could have been created as writable but later switched to FILE_STD_READ #endif }break; case FILE_MEM: if(_writable)Free(_mem); break; // write memory was dynamically allocated case FILE_MEMB: if(FILE_MEMB_UNION)DTOR(_memb);else _memb.clear(); break; } #if ANDROID if(_aasset)AAsset_close((AAsset*)_aasset); #endif zeroNoBuf(); } File& File::reset() { if(_type==FILE_MEMB) { _memb.reset(); _ok=true; _cipher_offset=0; _offset=_pos=_size=0; } return T; } File& File::append (C Str &name , Cipher *cipher) {if(!appendTry (name , cipher))Exit(MLT(S+"Can't append \""+ name+'"', PL,S+u"Nie można nadpisać \""+ name+'"')); return T;} File& File::write (C Str &name , Cipher *cipher) {if(! writeTry (name , cipher))Exit(MLT(S+"Can't create \""+ name+'"', PL,S+u"Nie można utworzyć \""+ name+'"')); return T;} File& File::readStd(C Str &name , Cipher *cipher) {if(! readStdTry(name , cipher))Exit(MLT(S+"Can't open \"" + name+'"', PL,S+u"Nie można otworzyć \""+ name+'"')); return T;} File& File::read (C PakFile &file, C Pak &pak ) {if(! readTry (file, pak ))Exit(MLT(S+"Can't open \"" + file.name+'"', PL,S+u"Nie można otworzyć \""+ file.name+'"')); return T;} File& File::read (C Str &name, C Pak &pak ) {if(! readTry (name, pak ))Exit(MLT(S+"Can't open \"" + name+'"', PL,S+u"Nie można otworzyć \""+ name+'"')); return T;} File& File::read (C UID &id , C Pak &pak ) {if(! readTry (id , pak ))Exit(MLT(S+"Can't open \"" +id.asCString()+'"', PL,S+u"Nie można otworzyć \""+id.asCString()+'"')); return T;} File& File::read (C Str &name, C PakSet &paks ) {if(! readTry (name, paks ))Exit(MLT(S+"Can't open \"" + name+'"', PL,S+u"Nie można otworzyć \""+ name+'"')); return T;} File& File::read (C UID &id , C PakSet &paks ) {if(! readTry (id , paks ))Exit(MLT(S+"Can't open \"" +id.asCString()+'"', PL,S+u"Nie można otworzyć \""+id.asCString()+'"')); return T;} File& File::read (C Str &name , Cipher *cipher) {if(! readTry (name , cipher))Exit(MLT(S+"Can't open \"" + name+'"', PL,S+u"Nie można otworzyć \""+ name+'"')); return T;} File& File::read (C UID &id , Cipher *cipher) {if(! readTry (id , cipher))Exit(MLT(S+"Can't open \"" +id.asCString()+'"', PL,S+u"Nie można otworzyć \""+id.asCString()+'"')); return T;} Bool File::copyToAndDiscard(Mems &dest) { if(T._type==FILE_MEM && T._writable && !_cipher && posAbs()==0) // check if we can just swap memories { // swap memories, perhaps 'File' may reuse 'Mems' data for future operations Ptr data=dest.data(); Int elms=dest.elms(); dest.setTemp((Byte*)T._mem, T._size); T._ok =true; T._cipher_offset=0; T._offset =0; T._pos =0; T._size =elms; T._cipher =null; T._mem =data; return true; }else { dest.setNum(left()); return getFast(dest.data(), dest.elms()); } } File& File::writeMemFixed(Int size, Cipher *cipher) { if(T._size==size && T._type==FILE_MEM && T._writable) // check if we can reuse existing memory { T._ok =true; T._cipher_offset=0; T._offset =0; T._pos =0; T._cipher =cipher; }else { close(); if(!size || (_mem=Alloc(size))) { T._type =FILE_MEM; T._writable=true; T._size =size; T._cipher =cipher; } } return T; } File& File::writeMem(UInt block_elms, Cipher *cipher) { if(T._type==FILE_MEMB && _memb.blockElms()==block_elms/* && T._writable - MEMB is always writable*/) { reset(); T._cipher=cipher; }else { close(); T._type =FILE_MEMB; T._writable=true; T._cipher =cipher; #if FILE_MEMB_UNION CTOR(T._memb, block_elms); #else if(_memb.blockElms()!=block_elms) { DTOR(_memb); CTOR(_memb, block_elms); } #endif } return T; } File& File::readMem(CPtr data, Int size, Cipher *cipher) { close(); if(size>=0) { T._type =FILE_MEM; T._writable=false; T._mem =(Ptr)data; T._size =size; T._cipher =cipher; } return T; } /******************************************************************************/ #if WINDOWS Bool File::appendTry (C Str &name, Cipher *cipher) {close(); if(name.is() && !_wsopen_s(&_handle, name, _O_BINARY|_O_RDWR|_O_CREAT , _SH_DENYWR, _S_IREAD|_S_IWRITE)){SetLastError(0); _type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _pos=_size=Max(0, _filelengthi64(_handle)); Seek(_handle, 0, SEEK_END); _cipher=cipher; return true;} _type=FILE_NONE; _close(_handle); _handle=0;} return false;} // clear error 183 file already exists Bool File:: writeTry (C Str &name, Cipher *cipher) {close(); if(name.is() && !_wsopen_s(&_handle, name, _O_BINARY|_O_RDWR|_O_CREAT|_O_TRUNC, _SH_DENYWR, _S_IREAD|_S_IWRITE)){SetLastError(0); _type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _cipher=cipher; return true;} _type=FILE_NONE; _close(_handle); _handle=0;} return false;} // clear error 183 file already exists Bool File:: readStdTryEx(C Str &name, Cipher *cipher, UInt max_buf_size) { close(); if(name.is()) { if(!_wsopen_s(&_handle, name, _O_BINARY|_O_RDONLY, _SH_DENYNO, _S_IREAD|_S_IWRITE)) { _type=FILE_STD_READ; _size=Max(0, _filelengthi64(_handle)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_CUR; _cipher=cipher; return true;} _type=FILE_NONE ; _close(_handle); _handle=0; _size=0; return false; } if(DataPath().is() && !FullPath(name)) { Char path_name[MAX_LONG_PATH]; MergePath(path_name, DataPath(), name); if(!_wsopen_s(&_handle, WChar(path_name), _O_BINARY|_O_RDONLY, _SH_DENYNO, _S_IREAD|_S_IWRITE)) { _type=FILE_STD_READ; _size=Max(0, _filelengthi64(_handle)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_DATA; _cipher=cipher; return true;} _type=FILE_NONE ; _close(_handle); _handle=0; _size=0; return false; } } } return false; } #elif APPLE static void DetectSymLink(File &f, C Str8 &unix_name) { struct stat stats; if(!fstat(f._handle, &stats))if(S_ISLNK(stats.st_mode)) { ASSERT(BUF_SIZE>=MAX_UTF_PATH); // we're reading directly to the File buffer, so make sure that it was set big enough to hold a link path ssize_t read=readlink(unix_name, (char*)f._buf, Min(f._buf_size, f.size())); if(read>0)f._buf_len=Int(read); } } Bool File::appendTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open(UnixPathUTF8(name), O_RDWR|O_CREAT| O_EXLOCK|O_NONBLOCK|O_SYMLINK, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _pos=_size=Max(0, Seek(_handle, 0, SEEK_END)); _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: writeTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open(UnixPathUTF8(name), O_RDWR|O_CREAT|O_TRUNC|O_EXLOCK|O_NONBLOCK|O_SYMLINK, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: readStdTryEx(C Str &name, Cipher *cipher, UInt max_buf_size) { close(); if(name.is()) { Str8 unix_name=UnixPathUTF8(name); _handle=open(unix_name, O_RDONLY|O_NONBLOCK|O_SYMLINK); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_CUR; Seek(_handle, 0, SEEK_SET); _cipher=cipher; DetectSymLink(T, unix_name); return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } if(DataPath().is() && !FullPath(name)) { Char path_name[MAX_LONG_PATH]; MergePath(path_name, DataPath(), name); unix_name=UnixPathUTF8(path_name); _handle=open(unix_name, O_RDONLY|O_NONBLOCK|O_SYMLINK); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_DATA; Seek(_handle, 0, SEEK_SET); _cipher=cipher; DetectSymLink(T, unix_name); return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } } } return false; } #elif LINUX || WEB Bool File::appendTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open64(UnixPathUTF8(name), O_RDWR|O_CREAT| O_NONBLOCK, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _pos=_size=Max(0, Seek(_handle, 0, SEEK_END)); _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: writeTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open64(UnixPathUTF8(name), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: readStdTryEx(C Str &name, Cipher *cipher, UInt max_buf_size) { close(); if(name.is()) { Str8 unix_name=UnixPathUTF8(name); _handle=open64(unix_name, O_RDONLY|O_NONBLOCK); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_CUR; Seek(_handle, 0, SEEK_SET); _cipher=cipher; return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } if(DataPath().is() && !FullPath(name)) { Char path_name[MAX_LONG_PATH]; MergePath(path_name, DataPath(), name); unix_name=UnixPathUTF8(path_name); _handle=open64(unix_name, O_RDONLY|O_NONBLOCK); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_DATA; Seek(_handle, 0, SEEK_SET); _cipher=cipher; return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } } } return false; } #elif ANDROID // Android has 'open64' only on API 21 and newer, however it's the same as 'open' with O_LARGEFILE Bool File::appendTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open(UnixPathUTF8(name), O_RDWR|O_CREAT| O_NONBLOCK|O_LARGEFILE, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _pos=_size=Max(0, Seek(_handle, 0, SEEK_END)); _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: writeTry (C Str &name, Cipher *cipher) {close(); if(name.is()){_handle=open(UnixPathUTF8(name), O_RDWR|O_CREAT|O_TRUNC|O_NONBLOCK|O_LARGEFILE, S_IRWXU|S_IRWXG|S_IRWXO); if(_handle>=0){_type=FILE_STD_WRITE; if(setBuf(BUF_SIZE)){_writable=true; _path=FILE_CUR; _cipher=cipher; return true;} _type=FILE_NONE; ::close(_handle); _handle=0;}} return false;} Bool File:: readStdTryEx(C Str &name, Cipher *cipher, UInt max_buf_size, Bool *processed) { if(processed)*processed=false; close(); if(name.is()) { // assets if(!FullPath(name) && AssetManager) if(AAsset *asset=AAssetManager_open(AssetManager, UnixPathUTF8(name), AASSET_MODE_RANDOM)) { Bool ok=false; #if __ANDROID_API__>=13 off64_t size=AAsset_getLength64(asset); #else off_t size=AAsset_getLength (asset); #endif if( size<0)ok=false;else if(!size ) // no need for opening the file using Android API when it has no size { T._type =FILE_MEM; T._writable=false; T._size =0; T._cipher =cipher; ok=true; }else // size>0 { #if __ANDROID_API__>=13 off64_t offset, length; int fd=AAsset_openFileDescriptor64(asset, &offset, &length); #else off_t offset, length; int fd=AAsset_openFileDescriptor (asset, &offset, &length); #endif if(fd>=0) { _size=length; if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))) { T._handle =fd; T._type =FILE_STD_READ; T._writable=false; T._path =FILE_ANDROID_ASSET; T._cipher =cipher; Seek(_handle, T._offset=offset, SEEK_SET); ok=true; }else { _size=0; ::close(fd); } }else // file is compressed, needs to be processed using 'AAsset_*' functions if(_mem=(Ptr)AAsset_getBuffer(asset)) { if(processed)*processed=true; // file had to be decompressed T._type =FILE_MEM; T._writable=false; T._path =FILE_ANDROID_ASSET; T._size =size; T._cipher =cipher; T._aasset =asset; return true; // return here and don't close 'asset' because that would make the '_mem' buffer invalid } } AAsset_close(asset); return ok; } // original path _handle=open(UnixPathUTF8(name), O_RDONLY|O_NONBLOCK|O_LARGEFILE); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_CUR; Seek(_handle, 0, SEEK_SET); _cipher=cipher; return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } // data path if(DataPath().is() && !FullPath(name)) { Char path_name[MAX_LONG_PATH]; MergePath(path_name, DataPath(), name); _handle=open(UnixPathUTF8(path_name), O_RDONLY|O_NONBLOCK|O_LARGEFILE); if(_handle>=0) { _type=FILE_STD_READ; _size=Max(0, Seek(_handle, 0, SEEK_END)); if(setBuf(Min(max_buf_size, Min(BUF_SIZE, _size)))){_writable=false; _path=FILE_DATA; Seek(_handle, 0, SEEK_SET); _cipher=cipher; return true;} _type=FILE_NONE ; ::close(_handle); _handle=0; _size=0; return false; } } } return false; } #endif /******************************************************************************/ Bool File::readTryRaw(C PakFile &file, C Pak &pak) { if(!(file.flag&PF_REMOVED))switch(pak._file_type) { case FILE_MEM: { readMem(pak._data, file.data_size_compressed, pak._file_cipher); T._pak =&pak; T._offset = pak._data_offset+file.data_offset; T._cipher_offset=(pak._cipher_per_file ? 0 : pak._file_cipher_offset+file.data_offset); }return true; case FILE_STD_READ: { if(T._type==FILE_STD_READ && !T._writable && T._pak==&pak) // this 'File' object already points to some 'PakFile' in the same 'Pak' as requested, which means that it's using the same stdio file, and we can re-use it { _offset=pak._data_offset+file.data_offset; if(Seek(_handle, _offset, SEEK_SET)==_offset) { T._size=file.data_size_compressed; if(setBuf(Min(BUF_SIZE, T._size))) { clearBuf(); T._ok =true; T._pos =0; T._cipher =pak._file_cipher; // re-apply cipher because this could have been changed in 'readTryEx' T._cipher_offset=(pak._cipher_per_file ? 0 : pak._file_cipher_offset+file.data_offset); return true; } } }else if(readStdTryEx(pak.pakFileName(), pak._file_cipher, file.data_size_compressed)) { _offset=pak._data_offset+file.data_offset; if(Seek(_handle, _offset, SEEK_SET)==_offset) { T._pak =&pak; T._size =file.data_size_compressed; T._cipher_offset=(pak._cipher_per_file ? 0 : pak._file_cipher_offset+file.data_offset); return true; } } }break; } close(); return false; } Bool File::readTryEx(C PakFile &file, C Pak &pak, Cipher *cipher, Bool *processed) { if(readTryRaw(file, pak)) { Bool p=false; if(file.compression) // first decompress with original cipher { p=true; // we had to perform processing File temp; if(!DecompressRaw(T, temp, file.compression, file.data_size_compressed, file.data_size, true))goto error; Swap(T, temp); pos(0); } if(cipher) // if we want to use custom cipher on top of what's already available (don't check for 'cipher!=T._cipher' because we want to add new cipher on top of existing) { if(T._cipher) // get rid of existing cipher by decrypting it to a memory file { p=true; // we had to perform processing File temp; if(!copy(temp.writeMemFixed(size(), null)))goto error; // we can't apply 'cipher' here and later clear it, because that would encrypt it with 'cipher', however what we want is to decrypt it with 'cipher' Swap(T, temp); pos(0); }else _cipher_offset=0; // if there's no cipher yet, but there was '_cipher_offset', then we need to clear it, as it would affect the new cipher T._cipher=cipher; // here we can just set '_cipher' instead of calling 'cipher' method, because in both cases (decompress+decrypt) we're writing to memory files so these operations are not needed } // this method calls 'readTryRaw' which then calls 'readStdTryEx' which on Android can set 'processed' to true, however not in this case, because we're loading a 'PakFile' from 'Pak', and if the File for the 'Pak' had to be processed, then 'Pak' itself decompressed it into a memory based 'File' FILE_MEM inside 'Pak.load' if(processed)*processed=p; return true; error: close(); }//else close(); no need to call 'close' here, because 'readTryRaw' will already call it if(processed)*processed=false; return false; } Bool File::readTryEx(C Str &name, C PakSet &paks, Cipher *cipher, Bool *processed) { //if(name.is())) in most cases the 'name' is going to be specified { SafeSyncLockerEx locker(paks._lock); // required if we call 'File.read' and 'PakSet.add' in multiple threads simultaneously if(C PaksFile *file=paks.find(name)) { Pak &pak=*file->pak ; C PakFile &pf =*file->file; locker.off(); // now when references have been copied, we can unlock return readTryEx(pf, pak, cipher, processed); } } close(); if(processed)*processed=false; return false; } Bool File::readTryEx(C UID &id, C PakSet &paks, Cipher *cipher, Bool *processed) { //if(id.valid())) in most cases the 'id' is going to be specified { SafeSyncLockerEx locker(paks._lock); // required if we call 'File.read' and 'PakSet.add' in multiple threads simultaneously if(C PaksFile *file=paks.find(id)) { Pak &pak=*file->pak ; C PakFile &pf =*file->file; locker.off(); // now when references have been copied, we can unlock return readTryEx(pf, pak, cipher, processed); } } close(); if(processed)*processed=false; return false; } Bool File::readTryEx(C Str &name, Cipher *cipher, Bool *processed) { if(readTryEx (name, Paks, cipher, processed))return true; #if !ANDROID if(processed)*processed=false; return readStdTryEx(name, cipher); #else return readStdTryEx(name, cipher, UINT_MAX, processed); #endif } Bool File::readTryEx(C UID &id, Cipher *cipher, Bool *processed) { if(readTryEx ( id , Paks, cipher, processed))return true; #if !ANDROID if(processed)*processed=false; return readStdTryEx(_EncodeFileName(id), cipher); #else return readStdTryEx(_EncodeFileName(id), cipher, UINT_MAX, processed); #endif } Bool File::readStdTry(C Str &name, Cipher *cipher) {return readStdTryEx(name, cipher);} Bool File::readTry (C Str &name, Cipher *cipher) {return readTryEx (name, cipher, null);} Bool File::readTry (C UID &id , Cipher *cipher) {return readTryEx (id , cipher, null);} Bool File::readTry (C Str &name, C PakSet &paks ) {return readTryEx (name, paks, null , null);} Bool File::readTry (C UID &id , C PakSet &paks ) {return readTryEx (id , paks, null , null);} Bool File::readTry (C PakFile &file, C Pak &pak ) {return readTryEx (file, pak , null , null);} Bool File::readTry(C Str &name, C Pak &pak) { if(C PakFile *file=pak.find(name, false))return readTry(*file, pak); close(); return false; } Bool File::readTry(C UID &id, C Pak &pak) { if(C PakFile *file=pak.find(id, false))return readTry(*file, pak); close(); return false; } /******************************************************************************/ Ptr File::mem() { switch(_type) { case FILE_MEM: return memFast(); } return null; } Long File::posFile()C { switch(_type) { default : return pos(); case FILE_STD_READ : return pos()+_buf_len; // in read mode, the actual position is ahead case FILE_STD_WRITE: return pos()-_buf_len; // in write mode, the actual position is behind } } Bool File::pos(Long pos) { if(pos==T._pos)return true; if(pos>=0)switch(_type) { case FILE_STD_READ: { { Long delta =pos-T._pos; if( delta<=_buf_len // we skip forward and we still have data in the buffer && -delta<=_buf_pos) // we skip backward and we have previous data in the buffer { _buf_len-=delta; _buf_pos+=delta; T._pos=pos; return true; // don't seek (because file position is already in order with the buffer) } } clearBuf(); // if we're going to seek then we need to clear the buffer if any seek: Long abs_pos=Seek(_handle, _offset+pos, SEEK_SET); if(abs_pos<0)return false; T._pos =abs_pos-_offset; // set position to what was actually set, in case it's different than what requested }return T._pos==pos; case FILE_STD_WRITE: { if(_buf_len && !flushDo()) { //return false; alternative approach is to just return false, however since 'pos' was called, then we take it with higher priority, and try to do what was requested #if ALLOW_REFLUSH _ok=false; // error occurred clearBuf(); // if we're allowing re-flush then it means some data could be left in the buffer, however because we're seeking, then we need to always discard it, so it's not saved at a different position #endif } }goto seek; case FILE_MEM : case FILE_MEMB: T._pos=pos; return true; } return false; } /******************************************************************************/ // COMPRESS / DECOMPRESS /******************************************************************************/ File& File::cmpIntV(Int i) { Byte data[MaxCmpUIntVSize]; Int pos=0; Bool negative=(i<0); UInt u=(negative ? -(i+1) : i); data[pos++]=((u&63)|(negative<<6)|((u>=64)<<7)); // 0 if(u>=64) { u>>=6; data[pos++]=((u&127)|((u>=128)<<7)); // 1 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 2 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 3 if(u>=128) { u>>=7; data[pos++]=(u); // 4 } } } } put(data, pos); return T; } File& File::decIntV(Int &i) { Byte v; T>>v; Bool negative=((v>>6)&1); UInt u=(v&63); if(v&128) { T>>v; u|=((v&127)<<6); if(v&128) { T>>v; u|=((v&127)<<(6+7)); if(v&128) { T>>v; u|=((v&127)<<(6+7+7)); if(v&128) { T>>v; u|=(v<<(6+7+7+7)); } } } } i=(negative ? -1-Int(u) : Int(u)); return T; } File& File::cmpUIntV(UInt u) { Byte data[MaxCmpUIntVSize]; Int pos=0; data[pos++]=((u&127)|((u>=128)<<7)); if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); if(u>=128) { u>>=7; data[pos++]=(u); } } } } put(data, pos); return T; } File& File::decUIntV(UInt &u) { Byte v; T>>v; u=(v&127); if(v&128) { T>>v; u|=((v&127)<<7); if(v&128) { T>>v; u|=((v&127)<<(7+7)); if(v&128) { T>>v; u|=((v&127)<<(7+7+7)); if(v&128) { T>>v; u|=(v<<(7+7+7+7)); } } } } return T; } File& File::cmpLongV(Long l) { Byte data[MaxCmpULongVSize]; Int pos=0; Bool negative=(l<0); ULong u=(negative ? -(l+1) : l); data[pos++]=((u&63)|(negative<<6)|((u>=64)<<7)); // 0 if(u>=64) { u>>=6; data[pos++]=((u&127)|((u>=128)<<7)); // 1 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 2 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 3 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 4 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 5 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 6 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 7 if(u>=128) { u>>=7; data[pos++]=(u); // 8 } } } } } } } } put(data, pos); return T; } File& File::decLongV(Long &l) { Byte v; T>>v; Bool negative=((v>>6)&1); ULong u=(v&63); if(v&128) { T>>v; u|=((v&127)<<6); if(v&128) { T>>v; u|=((v&127)<<(6+7)); if(v&128) { T>>v; u|=((v&127)<<(6+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(6+7+7+7)); // << 27 requires U64 if(v&128) { T>>v; u|=(U64(v&127)<<(6+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(6+7+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(6+7+7+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v)<<(6+7+7+7+7+7+7+7)); } } } } } } } } l=(negative ? -1-Long(u) : Long(u)); return T; } File& File::cmpULongVMax(ULong u) { Byte data[MaxCmpULongVSize]; data[0]=((u&127)|(1<<7)); // 0 u>>=7; data[1]=((u&127)|(1<<7)); // 1 u>>=7; data[2]=((u&127)|(1<<7)); // 2 u>>=7; data[3]=((u&127)|(1<<7)); // 3 u>>=7; data[4]=((u&127)|(1<<7)); // 4 u>>=7; data[5]=((u&127)|(1<<7)); // 5 u>>=7; data[6]=((u&127)|(1<<7)); // 6 u>>=7; data[7]=((u&127)|(1<<7)); // 7 u>>=7; data[8]=( u ); // 8 put(data); return T; } File& File::cmpULongV(ULong u) { Byte data[MaxCmpULongVSize]; Int pos=0; data[pos++]=((u&127)|((u>=128)<<7)); // 0 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 1 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 2 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 3 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 4 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 5 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 6 if(u>=128) { u>>=7; data[pos++]=((u&127)|((u>=128)<<7)); // 7 if(u>=128) { u>>=7; data[pos++]=(u); // 8 } } } } } } } } put(data, pos); return T; } File& File::decULongV(ULong &u) { Byte v; T>>v; u=(v&127); if(v&128) { T>>v; u|=((v&127)<<7); if(v&128) { T>>v; u|=((v&127)<<(7+7)); if(v&128) { T>>v; u|=((v&127)<<(7+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(7+7+7+7)); // << 28 requires U64 if(v&128) { T>>v; u|=(U64(v&127)<<(7+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(7+7+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v&127)<<(7+7+7+7+7+7+7)); if(v&128) { T>>v; u|=(U64(v)<<(7+7+7+7+7+7+7+7)); } } } } } } } } return T; } File& File::cmpFlt3cm(C Flt &r) {UInt u=Mid(Round(r*100)+8388608, 0, 16777216-1); put(&u, 3); ASSERT(SIZE(u)>=3); return T;} File& File::decFlt3cm( Flt &r) {Int i=0; get(&i, 3); r=(i-8388608)/100.0f; ASSERT(SIZE(i)>=3); return T;} File& File::cmpSatFlt1(C Flt &r) { putByte ( FltToByte(r) ); return T;} File& File::cmpSatFlt2(C Flt &r) { putUShort(RoundU(Sat(r)*65535)); return T;} // it's better to clamp flt for bigger values File& File::decSatFlt1( Flt &r) {r=getByte ()/ 255.0f; return T;} File& File::decSatFlt2( Flt &r) {r=getUShort()/65535.0f; return T;} File& File::cmpAngle1(C Flt &r) { putByte (RoundU(AngleFull(r)*( 256/PI2))); return T;} // use 'AngleFull' to make angle smaller and thus 'RoundPos' work with better precision, there's no need for 'Mid' or "&" because 1) AngleFull already sets 0..PI2 range, 2) putByte takes only Byte as param File& File::cmpAngle2(C Flt &r) { putUShort(RoundU(AngleFull(r)*(65536/PI2))); return T;} // use 'AngleFull' to make angle smaller and thus 'RoundPos' work with better precision, there's no need for 'Mid' or "&" because 1) AngleFull already sets 0..PI2 range, 2) putUShort takes only UShort as param File& File::decAngle1( Flt &r) {r=getByte ()*(PI2/ 256); return T;} File& File::decAngle2( Flt &r) {r=getUShort()*(PI2/65536); return T;} File& File::cmpDir2(C Vec &v) { Byte x=Mid(RoundPos(v.x*127+128), 0, 255), // 8-bits for X y=Mid(RoundPos(v.y* 63+ 64), 0, 127); // 7-bits for Y Bool z=(v.z<0); // 1-bit for Z sign putUShort(x|(y<<8)|(z<<15)); return T; } File& File::cmpDir3(C Vec &v) { UInt x=Mid(RoundPos(v.x*2047+2048), 0, 4095), // 12-bits for X y=Mid(RoundPos(v.y*1023+1024), 0, 2047); // 11-bits for Y Bool z=(v.z<0); // 1-bit for Z sign UInt c=(x|(y<<12)|(z<<23)); put(&c, 3); ASSERT(SIZE(c)>=3); return T; } File& File::decDir2(Vec &v) { UShort c=getUShort(); Int x=((c )&255), y=((c>> 8)&127); Bool z=((c>>15)& 1); v.x=(x-128)/127.0f; v.y=(y- 64)/ 63.0f; v.z=CalcZ(v.xy); if(z)CHS(v.z); v.normalize(); return T; } File& File::decDir3(Vec &v) { UInt c; get(&c, 3); ASSERT(SIZE(c)>=3); // clearing 'c' to zero first is not needed, since we're reading bits only from first 3 bytes below Int x=((c )&4095), y=((c>>12)&2047); Bool z=((c>>23)& 1); v.x=(x-2048)/2047.0f; v.y=(y-1024)/1023.0f; v.z=CalcZ(v.xy); if(z)CHS(v.z); v.normalize(); return T; } File& File::cmpOrient2(C Matrix3 &m) { Vec angles=m.angles(); Byte x=(Round(angles.x*(32/PI2))&31), // 5-bits for X y=(Round(angles.y*(64/PI2))&63), // 6-bits for Y (because most objects are on ground, and just rotated along Y axis, so give more precision to that axis) z=(Round(angles.z*(32/PI2))&31); // 5-bits for Z putUShort(x|(y<<5)|(z<<11)); return T; } File& File::cmpOrient3(C Matrix3 &m) { Vec angles=m.angles(); Byte b[]={Byte(Round(angles.x*(256/PI2))), Byte(Round(angles.y*(256/PI2))), Byte(Round(angles.z*(256/PI2)))}; T<> 5)&63), z=( c>>11 ); m.setRotateZ(z*(PI2/32)).rotateXY(x*(PI2/32), y*(PI2/64)); return T; } File& File::decOrient3(Matrix3 &m) { Byte b[3]; T>>b; m.setRotateZ(b[2]*(PI2/256)).rotateXY(b[0]*(PI2/256), b[1]*(PI2/256)); return T; } File& File::decOrient4(Matrix3 &m) { UInt c=getUInt(), x=( c &2047), y=((c>>11)&2047), z=( c>>22 ); m.setRotateZ(z*(PI2/1024)).rotateXY(x*(PI2/2048), y*(PI2/2048)); return T; } /******************************************************************************/ // MISC /******************************************************************************/ UInt File::memUsage()C { UInt mem=_buf_size; switch(_type) { case FILE_MEM : if(_writable)mem+=_size ; break; #if FILE_MEMB_UNION case FILE_MEMB: mem+=_memb.memUsage(); break; #endif } #if !FILE_MEMB_UNION mem+=_memb.memUsage(); #endif return mem; } /******************************************************************************/ Bool File::flushDo() { DEBUG_ASSERT(_type==FILE_STD_WRITE && _buf_len, "File.flushDo"); Int written =Write(_handle, _buf, _buf_len); if( written==_buf_len){_buf_len=0; return true;} // check this first because most likely this will happen, so return early #if ALLOW_REFLUSH if(written>0) // this avoids errors (<0) and when no data was written (==0) { _buf_len-=written; // decrease what we've written MoveFast(_buf, (Byte*)_buf+written, _buf_len); // since writing always uses start of the buffer, we need to copy remaining data at the start } // here '_pos, _buf_len, _ok' shouldn't be adjusted, because no data is lost, some was written and remaining is still in the buffer #else if(written>0)_pos+=written; // this avoids errors (<0) and when no data was written (==0) _pos-=_buf_len; // normally '_pos' is located already ahead at '_buf_len' position (assumes that everything was written), but if we've written less, then we need to set it back to what was lost _buf_len=0; // discard data _ok=false; // data was lost #endif return false; } Bool File::flushOK() {return flush() && ok();} Bool File::flush () { if(_buf_len && _type==FILE_STD_WRITE)return flushDo(); return true; } Bool File::sync() { if(_writable && (_type==FILE_STD_WRITE || _type==FILE_STD_READ)) // check 'FILE_STD_READ' too, because we may have written data previously in FILE_STD_WRITE, and later switched to FILE_STD_READ { #if WINDOWS #if 1 // faster return FlushFileBuffers((HANDLE)_get_osfhandle(_handle)); #else // internally calls 'FlushFileBuffers' which means this has bigger overhead return _commit(_handle)==0; #endif #else return fsync(_handle)==0; #endif } return true; } /******************************************************************************/ void File::cipher(Cipher *cipher) { if(T._cipher!=cipher) { if(_type==FILE_STD_READ) // if we're in reading mode (we can skip FILE_STD_WRITE because any data saved in the buffer is saved with correct cipher already, and we don't want to change it) if(Int buf_size=_buf_pos+_buf_len) // and we have any data in buffer that we've read ahead { auto offset=posCipher()-_buf_pos; if(T._cipher)T._cipher->encrypt(_buf, _buf, buf_size, offset); // we have to encrypt entire buffer back using old cipher if( cipher) cipher->decrypt(_buf, _buf, buf_size, offset); // decrypt using new cipher } T._cipher=cipher; } } void File::cipherOffset(Int offset) { if(T._cipher_offset!=offset) { if(_type==FILE_STD_READ) // if we're in reading mode (we can skip FILE_STD_WRITE because any data saved in the buffer is saved with correct cipher offset already, and we don't want to change it) if(_cipher) // we have any cipher if(Int buf_size=_buf_pos+_buf_len) // and we have any data in buffer that we've read ahead { _cipher->encrypt(_buf, _buf, buf_size, posCipher()-_buf_pos); T._cipher_offset=offset; // we have to encrypt entire buffer back using old cipher offset _cipher->decrypt(_buf, _buf, buf_size, posCipher()-_buf_pos); // decrypt using new cipher offset } T._cipher_offset=offset; } } /******************************************************************************/ Bool File::size(Long size) { if(size==T.size())return true; if(size<0 || _offset)return false; if(!flush() && ALLOW_REFLUSH)_ok=false; // if flush failed, then set as ok=false, because the data will be discarded switch(_type) { case FILE_STD_READ : case FILE_STD_WRITE: //if(_writable) we can skip this check because the system calls will just fail for non-writable { #if WINDOWS if(!_chsize_s (_handle, size)) #elif APPLE || (ANDROID && __ANDROID_API__<12) // on Apple 'ftruncate' is already 64-bit, while Android has 'ftruncate64' only on API 12 and above if(!ftruncate (_handle, size)) #else if(!ftruncate64(_handle, size)) #endif { // no need to call 'setBuf' because we can only change size for files in write mode, an in this mode files always have BUF_SIZE buffer // no need to clip current position to size limit, because 'File' allows position being after the end _size=size; discardBuf(false); // disable flushing because we've already tried it at the start, and doing it again could change the file size } }break; case FILE_MEM: { if(_writable)_Realloc(_mem, size, _size); // write mode allocated memory manually, so we need to reallocate it _size=size; }break; case FILE_MEMB: _size=_memb.setNum(size).elms(); break; // set '_size' from what 'Memb' will actually have } return _size==size; } /******************************************************************************/ Int File::getReturnSize(Ptr data, Int size) { Long old_pos=_pos; if(!data)skip(size);else { MIN(size, left()); if (size>0)switch(_type) { case FILE_STD_WRITE: { if(!discardBuf(true))break; // if this fails then we can't read, because we're expecting to read at the current position _type=FILE_STD_READ; // set new mode and fall down for reading } // !! no break on purpose !! case FILE_STD_READ : { if(_buf_len) // have data in the buffer { get_from_buffer: Int l=Min(_buf_len, size); CopyFast(data, (Byte*)_buf+_buf_pos, l); _buf_pos+=l; _buf_len-=l; _pos +=l; size-=l; if(size<=0)break; data =(Byte*)data+l; // can be placed after break } // read from file if(size>=_buf_size/2) // in this case it will be faster to don't use buffering and just read directly to target memory // "size>=_buf_size" was tested as well, but it was slower { // read to 'data' Int l =Read(_handle, data, size); if( l<=0)break; if(_cipher)_cipher->decrypt(data, data, l, posCipher()); _pos +=l; size-=l; if(size<=0)break; data =(Byte*)data+l; // can be placed after break } // read to buffer Int l =Read(_handle, _buf, Min(_buf_size, left())); // read as much as possible if( l<=0)break; if(_cipher)_cipher->decrypt(_buf, _buf, l, posCipher()); // decrypt entire buffer at the same time (faster) _buf_pos=0; _buf_len=l; goto get_from_buffer; }break; case FILE_MEM: { Ptr src=memFast(); if(_cipher)_cipher->decrypt(data, src, size, posCipher()); else CopyFast(data, src, size ); _pos+=size; //data=(Byte*)data+size; not needed //size= 0; not needed }break; case FILE_MEMB: { for(;;) { Int pos_abs=posAbs(), // Long not needed because later this is used only as Int memb element index elm =pos_abs & _memb.mask(), // index of element in a block l =Min(size, _memb.blockElms()-elm); // how much elements left in this block if(_cipher)_cipher->decrypt(data, &_memb[pos_abs], l, posCipher()); else CopyFast(data, &_memb[pos_abs], l ); _pos +=l; size-=l; if(size<=0)break; data =(Byte*)data+l; // can be placed after break } }break; } } return _pos-old_pos; // return number of bytes read } Int File::putReturnSize(CPtr data, Int size) { Long old_pos=_pos; if(size>0) { switch(_type) { case FILE_STD_READ: { if(!_writable || !discardBuf(false))break; // if this fails then we can't write, because we're expecting to write at the current position, no need for flush because FILE_STD_READ doesn't need flush _type=FILE_STD_WRITE; // set new mode and fall down for writing } // !! no break on purpose !! case FILE_STD_WRITE: { if(!_cipher && data) // buffering optional (check for 'data' because it's allowed to be null and write zeros) { do{ if(_buf_len || size<_buf_size) // copy to buffer { Int l=Min(size, _buf_size-_buf_len); CopyFast((Byte*)_buf+_buf_len, data, l); _buf_len+=l; data =(Byte*)data+l; _pos +=l; size-=l; if(_buf_len>=_buf_size && !flushDo())break; // if buffer is full then flush and check for errors }else // direct write { Int written=Write(_handle, data, size); if(written<=0)break; data =(Byte*)data+written; _pos +=written; size-=written; } }while(size>0); }else // buffering necessary { do{ // copy to buffer Int l=Min(size, _buf_size-_buf_len); if(_cipher)_cipher->encrypt((Byte*)_buf+_buf_len, data, l, posCipher()); else Copy((Byte*)_buf+_buf_len, data, l ); _buf_len+=l; if(data)data=(Byte*)data+l; _pos +=l; size-=l; if(_buf_len>=_buf_size && !flushDo())break; // if buffer is full then flush and check for errors }while(size>0); } }break; case FILE_MEM: if(_writable) { Int write=Min(size, left()); // don't write more than is allocated Ptr dest =memFast(); if(_cipher)_cipher->encrypt(dest, data, write, posCipher()); else Copy(dest, data, write ); _pos +=write; size-=write; }break; case FILE_MEMB: { Int dest_elms=posAbs()+size; if(dest_elms>_memb.elms())_memb.setNum(dest_elms); // Long not needed because is used for Memb based on 32-bits do{ Int pos_abs=posAbs(), // Long not needed because later is used only as Int memb element index elm =pos_abs & _memb.mask(), // index of element in a block l =Min(size, _memb.blockElms()-elm); // how much elements left in this block if(_cipher)_cipher->encrypt(&_memb[pos_abs], data, l, posCipher()); else Copy(&_memb[pos_abs], data, l ); if(data)data=(Byte*)data+l; _pos +=l; size-=l; }while(size>0); }break; } MAX(_size, _pos); } return _pos-old_pos; // return number of bytes written } Bool File::getFast(Ptr data, Int size) { if(getReturnSize(data, size)==size)return true; _ok=false; return false; // set error } Bool File::get(Ptr data, Int size) { Int read=getReturnSize(data, size); if( read==size)return true; // check this first because this is what's most likely going to happen if( data)Zero((Byte*)data+read, size-read); // zero unread data, this is important because methods such as f.getInt, f.decUIntV, .. don't check for status, however they're expected to return zeros _ok=false; return false; // set error } Bool File::put(CPtr data, Int size) { Int written=putReturnSize(data, size); if( written==size)return true; // check this first because this is what's most likely going to happen _ok=false; return false; // set error } /******************************************************************************/ File& File::putStr(CChar8 *t) { Int length =Length (t); Bool unicode=HasUnicode(t); cmpIntV(unicode ? -length : length); if(length) { if(unicode) { Memt temp; temp.setNum(length+1); Set(temp.data(), t, temp.elms()); putN(temp.data(), length); }else { putN(t, length); } } return T; } File& File::putStr(CChar *t) { Int length =Length (t); Bool unicode=HasUnicode(t); cmpIntV(unicode ? -length : length); if(length) { if(unicode) { putN(t, length); }else { Memt temp; temp.setNum(length+1); Set(temp.data(), t, temp.elms()); putN(temp.data(), length); } } return T; } File& File::putStr(C Str8 &s) // keep this function to allow having '\0' chars in the middle { Int length =s.length(); Bool unicode=HasUnicode(s); cmpIntV(unicode ? -length : length); if(length) { if(unicode){Memt temp; temp.setNum(length); FREPAO(temp)=Char8To16Fast(s[i]); putN(temp.data(), length);} // we can assume that Str was already initialized else { putN( s(), length);} } return T; } File& File::putStr(C Str &s) // keep this function to allow having '\0' chars in the middle { Int length =s.length(); Bool unicode=HasUnicode(s); cmpIntV(unicode ? -length : length); if(length) { if(unicode){ putN( s(), length);} else {Memt temp; temp.setNum(length); FREPAO(temp)=Char16To8Fast(s[i]); putN(temp.data(), length);} // we can assume that Str was already initialized } return T; } File& File::skipStr() { Int length; decIntV(length); if( length<0){CHS(length); length*=SIZE(Char );} // unicode else length*=SIZE(Char8); if(left() temp; temp.setNum(length); getN(temp.data(), length); FREPA(temp)s._d[i]=Char16To8Fast(temp[i]); s._d[s._length=length]=0; // we can assume that Str was already initialized }else if(length) { if(left()0) { Int read=Min(length, t_elms-1); Memt temp; temp.setNum(read); getN(temp.data(), read); FREP(read)t[i]=Char16To8Fast(temp[i]); t[read]=0; // we can assume that Str was already initialized length-=read; } length*=SIZE(Char); }else { if(left()0) { Int read=Min(length, t_elms-1); getN(t, read); t[read]=0; length-=read; } } skip(length); if(ok())return T; goto error; length_too_long: _ok=false; pos(size()); // if the length was too long then go at the end of file, in case the user will try to read more data after this call, this is important so that the partially available string data is not treated as something else error: if(t && t_elms>0)t[0]='\0'; return T; } File& File::getStr(Char *t, Int t_elms) { Int length; decIntV(length); if( length<0) // unicode { CHS(length); if(left()0) { Int read=Min(length, t_elms-1); getN(t, read); t[read]=0; length-=read; } length*=SIZE(Char); }else { if(left()0) { Int read=Min(length, t_elms-1); Char8 *temp=(Char8*)t; getN(temp, read); // we can re-use the char array memory because it uses Char which has 2x Char8 capacity t[read]=0; // because we're processing from the end, then start with the end too REP(read)t[i]=Char8To16Fast(temp[i]); // need to process from the end to not overwrite the source, we can assume that Str was already initialized length-=read; } } skip(length); if(ok())return T; goto error; length_too_long: _ok=false; pos(size()); // if the length was too long then go at the end of file, in case the user will try to read more data after this call, this is important so that the partially available string data is not treated as something else error: if(t && t_elms>0)t[0]='\0'; return T; } Str File::getStr() {Str s; getStr(s); return s;} // warning: this must handle having '\0' chars in the middle /******************************************************************************/ File& File::putAsset(C UID &id) { Int used; REPAS(used, id.b)if(id.b[used])break; used++; putByte(used).put(id.b, used); return T; } File& File::putAsset(CChar *t) { if(!Is(t))putByte(0);else { UID id; if(DecodeFileName(t, id))putAsset(id);else putByte(0xFF).putStr(t); } return T; } UID File::getAssetID() { Byte b; T>>b; if( b<=SIZE(UID)) { UID id; if(getFast(id.b, b)) { Byte *d=id.b+b; REP(SIZE(id)-b)d[i]=0; return id; } }else if(b==0xFF)skipStr(); //else error return UIDZero; } File& File::getAsset(Str &s) { Byte b; T>>b; if( b<=SIZE(UID)) { if(!b)s.clear();else { UID id; if(getFast(id.b, b)) { Byte *d=id.b+b; REP(SIZE(id)-b)d[i]=0; s=_EncodeFileName(id); }else s.clear(); } }else if(b==0xFF)getStr(s); else s.clear(); // error return T; } Str File::getAsset() {Str s; getAsset(s); return s;} /******************************************************************************/ // Deprecated, do not use File& File::_decIntV(Int &i) { Byte v; T>>v; Bool positive=((v>>6)&1); UInt u=(v&63); if(v&128) { T>>v; u|=((v&127)<<6); if(v&128) { T>>v; u|=((v&127)<<(6+7)); if(v&128) { T>>v; u|=((v&127)<<(6+7+7)); if(v&128) { T>>v; u|=(v<<(6+7+7+7)); } } } } i=(positive ? u+1 : -Int(u)); return T; } Str File::_getStr2( ) {Str s; _getStr2(s); return s;} // warning: this must handle having '\0' chars in the middle File& File::_getStr2(Str &s) // warning: this must handle having '\0' chars in the middle { s.clear(); // always 'clear' even for 'reserve' to avoid copying old data in 'setNum' Int length; _decIntV(length); if( length<0) // unicode { CHS(length); if(left() temp; temp.setNum(length); getN(temp.data(), length); FREPA(temp)s._d[i]=Char16To8Fast(temp[i]); s._d[s._length=length]=0; // we can assume that Str was already initialized }else if(length) { if(left()0) { Int read=Min(length, t_elms-1); Memt temp; temp.setNum(read); getN(temp.data(), read); FREP(read)t[i]=Char16To8Fast(temp[i]); t[read]=0; // we can assume that Str was already initialized length-=read; } length*=SIZE(Char); }else { if(left()0) { Int read=Min(length, t_elms-1); getN(t, read); t[read]=0; length-=read; } } skip(length); if(ok())return T; goto error; length_too_long: _ok=false; pos(size()); // if the length was too long then go at the end of file, in case the user will try to read more data after this call, this is important so that the partially available string data is not treated as something else error: if(t && t_elms>0)t[0]='\0'; return T; } File& File::_getStr2(Char *t, Int t_elms) { Int length; _decIntV(length); if( length<0) // unicode { CHS(length); if(left()0) { Int read=Min(length, t_elms-1); getN(t, read); t[read]=0; length-=read; } length*=SIZE(Char); }else { if(left()0) { Int read=Min(length, t_elms-1); Char8 *temp=(Char8*)t; getN(temp, read); // we can re-use the char array memory because it uses Char which has 2x Char8 capacity t[read]=0; // because we're processing from the end, then start with the end too REP(read)t[i]=Char8To16Fast(temp[i]); // need to process from the end to not overwrite the source, we can assume that Str was already initialized length-=read; } } skip(length); if(ok())return T; goto error; length_too_long: _ok=false; pos(size()); // if the length was too long then go at the end of file, in case the user will try to read more data after this call, this is important so that the partially available string data is not treated as something else error: if(t && t_elms>0)t[0]='\0'; return T; } File& File::_putStr(C Str8 &s) // warning: this must handle having '\0' chars in the middle { UInt length =s.length(); Bool unicode=HasUnicode(s); putUInt(unicode ? length^SIGN_BIT : length); if(length) { if(unicode){Memt temp; temp.setNum(length); FREPAO(temp)=Char8To16Fast(s[i]); putN(temp.data(), length);} // we can assume that Str was already initialized else { putN( s(), length);} } return T; } File& File::_putStr(C Str &s) // warning: this must handle having '\0' chars in the middle { UInt length =s.length(); Bool unicode=HasUnicode(s); putUInt(unicode ? length^SIGN_BIT : length); if(length) { if(unicode){ putN( s(), length);} else {Memt temp; temp.setNum(length); FREPAO(temp)=Char16To8Fast(s[i]); putN(temp.data(), length);} // we can assume that Str was already initialized } return T; } File& File::_getStr(Str8 &s) // warning: this must handle having '\0' chars in the middle { s.clear(); // always 'clear' even for 'reserve' to avoid copying old data in 'setNum' UInt length=getUInt(); if( length&SIGN_BIT) // unicode { length^=SIGN_BIT; MIN(length, left()/SIZEI(Char)); if(length) { s.reserve(length); Memt temp; temp.setNum(length); getN(temp.data(), length); FREPA(temp)s._d[i]=Char16To8Fast(temp[i]); // we can assume that Str was already initialized s._d[s._length=length]=0; } }else { MIN(length, left()); if (length) { s.reserve(length); getN(s._d.data(), length); s._d[s._length=length]=0; } } return T; } Str File::_getStr( ) {Str s; _getStr(s); return s;} // warning: this must handle having '\0' chars in the middle File& File::_getStr(Str &s) // warning: this must handle having '\0' chars in the middle { s.clear(); // always 'clear' even for 'reserve' to avoid copying old data in 'setNum' UInt length=getUInt(); if( length&SIGN_BIT) // unicode { length^=SIGN_BIT; MIN(length, left()/SIZEI(Char)); if(length) { s.reserve(length); getN(s._d.data(), length); s._d[s._length=length]=0; } }else { MIN(length, left()); if (length) { s.reserve(length); Memt temp; temp.setNum(length); getN(temp.data(), length); FREPA(temp)s._d[i]=Char8To16Fast(temp[i]); // we can assume that Str was already initialized s._d[s._length=length]=0; } } return T; } File& File::_getStr(Char *t, Int t_elms) { UInt length=getUInt(); if( length&SIGN_BIT) // unicode { length^=SIGN_BIT; if(left()0) { Int read=Min(length, t_elms-1); getN(t, read); t[read]=0; length-=read; } length*=SIZE(Char); }else { if(left()0) { Int read=Min(length, t_elms-1); Char8 *temp=(Char8*)t; getN(temp, read); // we can re-use the char array memory because it uses Char which has 2x Char8 capacity t[read]=0; // because we're processing from the end, then start with the end too REP(read)t[i]=Char8To16Fast(temp[i]); // need to process from the end to not overwrite the source, we can assume that Str was already initialized length-=read; } } skip(length); if(ok())return T; goto error; length_too_long: _ok=false; pos(size()); // if the length was too long then go at the end of file, in case the user will try to read more data after this call, this is important so that the partially available string data is not treated as something else error: if(t && t_elms>0)t[0]='\0'; return T; } Str8 File::_getStr8() { Str8 s; UInt length=getUInt(); s._d.setNum( length+1); getN(s._d.data(), length+1); s._length = length ; return s; } Str File::_getStr16() { Str s; UInt length=getUInt(); s._d.setNum( length+1); getN(s._d.data(), length+1); s._length = length ; return s; } File& File::_putAsset(CChar *t) { if(!Is(t))putByte(0);else {UID id; if(DecodeFileName(t, id))putByte(2)<>id; return _EncodeFileName(id);} } } /******************************************************************************/ Short File::getBEShort () { Short i ; T>>i; SwapEndian(i); return i;} UShort File::getBEUShort() {UShort i ; T>>i; SwapEndian(i); return i;} Int File::getBEInt24 () {Int24 i ; T>>i; Swap(i.b[0], i.b[2]); return i.asInt();} UInt File::getBEUInt24() {Byte b[3]; T>>b; return b[2] | (b[1]<<8) | (b[0]<<16);} Int File::getBEInt () { Int i ; T>>i; SwapEndian(i); return i;} UInt File::getBEUInt () {UInt i ; T>>i; SwapEndian(i); return i;} Long File::getBELong () { Long i ; T>>i; SwapEndian(i); return i;} ULong File::getBEULong () {ULong i ; T>>i; SwapEndian(i); return i;} /******************************************************************************/ Bool File::equal(File &f, Long max_size) { if(max_size<0 || left()0) { Memt temp; temp.setNum(TEMP_BUF_SIZE); Int buf_size=temp.elms()/2; Ptr buf=temp.data(), buf2=temp.data()+buf_size; for(; max_size>0; ) { Int l=Min(buf_size, max_size); max_size-=l; if(!getFast (buf, l) || !f.getFast(buf2, l))return false; if(!EqualMem(buf, buf2, l))return false; } } return true; } Bool File::copy(File &dest, Long max_size) { if(max_size<0)max_size=left(); if(max_size>0) { Memt temp; temp.setNum(TEMP_BUF_SIZE); Ptr buf=temp.data(); Int buf_size=temp.elms(); REP( max_size/buf_size )if(!getFast(buf, buf_size) || !dest.put(buf, buf_size))return false; buf_size=max_size%buf_size; if(!getFast(buf, buf_size) || !dest.put(buf, buf_size))return false; } return true; } Bool File::copy(File &dest, DataCallback &callback, Long max_size) { if(max_size<0)max_size=left(); if(max_size>0) { Memt temp; temp.setNum(TEMP_BUF_SIZE); Ptr buf=temp.data(); Int buf_size=temp.elms(); REP( max_size/buf_size ){if(!getFast(buf, buf_size) || !dest.put(buf, buf_size))return false; callback.data(buf, buf_size);} buf_size=max_size%buf_size; if(!getFast(buf, buf_size) || !dest.put(buf, buf_size))return false; callback.data(buf, buf_size); } return true; } Bool File::copyEncrypt(File &dest, Cipher &cipher, Long max_size) { if(max_size<0)max_size=left(); if(max_size>0) { Memt temp; temp.setNum(TEMP_BUF_SIZE); Ptr buf=temp.data(); Int buf_size=temp.elms(), offset=0; REP( max_size/buf_size ){if(!getFast(buf, buf_size))return false; cipher.encrypt(buf, buf, buf_size, offset); if(!dest.put(buf, buf_size))return false; offset+=buf_size;} buf_size=max_size%buf_size; if(!getFast(buf, buf_size))return false; cipher.encrypt(buf, buf, buf_size, offset); if(!dest.put(buf, buf_size))return false; } return true; } /******************************************************************************/ void File:: clearBuf() {_buf_pos=_buf_len=0;} // !! this does not reset the system file handle position, if the buffer read something, then the position is ahead !! Bool File::discardBuf(Bool flush) { if(_buf_pos || _buf_len) // if there's any data in the buffer { Long pos=T.pos(); // remember current position before doing any operation if(flush && !T.flush() && ALLOW_REFLUSH)_ok=false; // !! do this after remembering position because this method may change it !! if we had some data to save which failed, then only disable '_ok', but still proceed because here the priority is to set correct file position T._pos=posFile(); // set actual position, so calling "T.pos(pos)" will proceed because current position is different than desired clearBuf(); // !! do this after calling 'posFile' !! always clear buffer (in case read mode or in case write mode flush fail) return T.pos(pos); // set remembered position } return true; } /******************************************************************************/ void File:: limit(ULong &total_size, ULong &applied_offset, Long new_size) {total_size=size(); applied_offset=pos(); T._size= new_size; T._offset+=applied_offset; T._cipher_offset+=applied_offset; T._pos-=applied_offset;} void File::unlimit(ULong &total_size, ULong &applied_offset ) { T._size=total_size; T._offset-=applied_offset; T._cipher_offset-=applied_offset; T._pos+=applied_offset; applied_offset=0;} /******************************************************************************/ UInt File::crc32(Long max_size) { Byte buf[TEMP_BUF_SIZE]; CRC32 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return 0; hash.update(buf, left); max_size-=left; } return hash(); } UInt File::xxHash32(Long max_size) { Byte buf[TEMP_BUF_SIZE]; ::xxHash32 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return 0; hash.update(buf, left); max_size-=left; } return hash(); } #pragma runtime_checks("", off) UInt File::xxHash64_32(Long max_size) {return xxHash64(max_size);} // we're interested only in low 32 bits #pragma runtime_checks("", restore) ULong File::xxHash64 (Long max_size) { Byte buf[TEMP_BUF_SIZE]; ::xxHash64 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return 0; hash.update(buf, left); max_size-=left; } return hash(); } UInt File::spookyHash32 (Long max_size) {return spookyHash128(max_size).i[0];} // can use part of 128-bit because this is how 'SpookyHash' works ULong File::spookyHash64 (Long max_size) {return spookyHash128(max_size).l[0];} // can use part of 128-bit because this is how 'SpookyHash' works UID File::spookyHash128(Long max_size) { Byte buf[TEMP_BUF_SIZE]; SpookyHash hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return UIDZero; hash.update(buf, left); max_size-=left; } return hash.hash128(); } ULong File::metroHash64(Long max_size) { Byte buf[TEMP_BUF_SIZE]; MetroHash64 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return 0; hash.update(buf, left); max_size-=left; } return hash(); } UID File::metroHash128(Long max_size) { Byte buf[TEMP_BUF_SIZE]; MetroHash128 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return UIDZero; hash.update(buf, left); max_size-=left; } return hash(); } UID File::md5(Long max_size) { Byte buf[TEMP_BUF_SIZE]; MD5 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left))return UIDZero; hash.update(buf, left); max_size-=left; } return hash(); } SHA1::Hash File::sha1(Long max_size) { Byte buf[TEMP_BUF_SIZE]; SHA1 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left)){hash._hash.zero(); return hash._hash;} hash.update(buf, left); max_size-=left; } return hash(); } SHA2::Hash File::sha2(Long max_size) { Byte buf[TEMP_BUF_SIZE]; SHA2 hash; if(max_size<0)max_size=left(); for(;;) { Int left=Min(SIZEI(buf), max_size); if(left<=0)break; if(!getFast(buf, left)){hash._hash.zero(); return hash._hash;} hash.update(buf, left); max_size-=left; } return hash(); } /******************************************************************************/ } /******************************************************************************/