Finalize LZ4 support. Do not repeatedly allocate compressed data buffer. Do not store global blocksize into the package file header. Increased Android read buffer to 32KB.

Docs/LuaScriptAPI.dox

@@ -2691,6 +2691,7 @@ Methods:
 - unsigned GetNumFiles() const
 - unsigned GetTotalSize() const
 - unsigned GetChecksum() const
+- bool IsCompressed() const
 
 Properties:
 
@@ -2699,6 +2700,7 @@ Properties:
 - unsigned numFiles (readonly)
 - unsigned totalSize (readonly)
 - unsigned checksum (readonly)
+- bool compressed (readonly)
 
 ### Serializer
 

Docs/Reference.dox

@@ -1702,7 +1702,7 @@ Note: outputting only bone rotations may help when using an animation in a diffe
 
 \section Tools_PackageTool PackageTool
 
-Examines a directory recursively for files and subdirectories and creates a PackageFile. The package file can be added to the ResourceCache and used as if the files were on a (read-only) filesystem.
+Examines a directory recursively for files and subdirectories and creates a PackageFile. The package file can be added to the ResourceCache and used as if the files were on a (read-only) filesystem. The file data can optionally be compressed using the LZ4 compression library.
 
 Usage:
 
@@ -1718,7 +1718,7 @@ For example, this would convert all the resource files inside the Urho3D Data di
 PackageTool Data Data.pak
 \endverbatim
 
-The -c option enables LZ4 file compression on the package.
+The -c option enables LZ4 compression on the files.
 
 \section Tools_RampGenerator RampGenerator
 
@@ -1927,7 +1927,6 @@ byte[]     Bytecode
 byte[4]    Identifier "UPAK" or "ULZ4" if compressed
 uint       Number of file entries
 uint       Whole package checksum
-(uint)     Compressed block size, only included if compressed
 
     For each file entry:
     cstring    Name
@@ -1935,7 +1934,7 @@ uint       Whole package checksum
     uint       Size
     uint       Checksum
 
-    For each compressed block in file data:
+    The compressed data for each file is the following, repeated until the file is done:
     ushort     Uncompressed length of block
     ushort     Compressed length of block
     byte[]     Compressed data

Docs/ScriptAPI.dox

@@ -1553,6 +1553,7 @@ Methods:
 - void SendEvent(const String&, VariantMap& = VariantMap ( ))
 - bool Open(const String&) const
 - bool Exists(const String&) const
+- bool compressed() const
 
 Properties:
 

Source/Engine/IO/File.cpp

@@ -52,7 +52,8 @@ static const char* openMode[] =
 };
 #endif
 
-static const unsigned READ_BUFFER_SIZE = 1024;
+static const unsigned READ_BUFFER_SIZE = 32768;
+static const unsigned SKIP_BUFFER_SIZE = 1024;
 
 File::File(Context* context) :
     Object(context),
@@ -64,7 +65,8 @@ File::File(Context* context) :
     readBufferOffset_(0),
     readBufferSize_(0),
     offset_(0),
-    checksum_(0)
+    checksum_(0),
+    compressed_(false)
 {
 }
 
@@ -78,7 +80,8 @@ File::File(Context* context, const String& fileName, FileMode mode) :
     readBufferOffset_(0),
     readBufferSize_(0),
     offset_(0),
-    checksum_(0)
+    checksum_(0),
+    compressed_(false)
 {
     Open(fileName, mode);
 }
@@ -93,7 +96,8 @@ File::File(Context* context, PackageFile* package, const String& fileName) :
     readBufferOffset_(0),
     readBufferSize_(0),
     offset_(0),
-    checksum_(0)
+    checksum_(0),
+    compressed_(false)
 {
     Open(package, fileName);
 }
@@ -162,6 +166,7 @@ bool File::Open(const String& fileName, FileMode mode)
     position_ = 0;
     offset_ = 0;
     checksum_ = 0;
+    compressed_ = false;
     
     fseek((FILE*)handle_, 0, SEEK_END);
     size_ = ftell((FILE*)handle_);
@@ -197,13 +202,7 @@ bool File::Open(PackageFile* package, const String& fileName)
     checksum_ = entry->checksum_;
     position_ = 0;
     size_ = entry->size_;
-    
-    if (package->IsCompressed())
-    {
-        readBuffer_ = new unsigned char[package->GetBlockSize()];
-        readBufferOffset_ = 0;
-        readBufferSize_ = 0;
-    }
+    compressed_ = package->IsCompressed();
     
     fseek((FILE*)handle_, offset_, SEEK_SET);
     return true;
@@ -247,16 +246,15 @@ unsigned File::Read(void* dest, unsigned size)
         
         return size;
     }
-    #else
-    // Package compressed mode
-    if (readBuffer_)
+    #endif
+    if (compressed_)
     {
         unsigned sizeLeft = size;
         unsigned char* destPtr = (unsigned char*)dest;
         
         while (sizeLeft)
         {
-            if (readBufferOffset_ >= readBufferSize_)
+            if (!readBuffer_ || readBufferOffset_ >= readBufferSize_)
             {
                 unsigned char blockHeaderBytes[4];
                 fread(blockHeaderBytes, sizeof blockHeaderBytes, 1, (FILE*)handle_);
@@ -265,11 +263,15 @@ unsigned File::Read(void* dest, unsigned size)
                 unsigned unpackedSize = blockHeader.ReadUShort();
                 unsigned packedSize = blockHeader.ReadUShort();
                 
-                /// \todo Should reuse a block to hold the packed data
+                if (!readBuffer_)
+                {
+                    readBuffer_ = new unsigned char[unpackedSize];
+                    inputBuffer_ = new unsigned char[LZ4_compressBound(unpackedSize)];
+                }
+                
                 /// \todo Handle errors
-                SharedArrayPtr<unsigned char> packedBytes(new unsigned char[packedSize]);
-                fread(packedBytes.Get(), packedSize, 1, (FILE*)handle_);
-                LZ4_decompress_fast((const char*)packedBytes.Get(), (char *)readBuffer_.Get(), unpackedSize);
+                fread(inputBuffer_.Get(), packedSize, 1, (FILE*)handle_);
+                LZ4_decompress_fast((const char*)inputBuffer_.Get(), (char *)readBuffer_.Get(), unpackedSize);
                 
                 readBufferSize_ = unpackedSize;
                 readBufferOffset_ = 0;
@@ -285,7 +287,6 @@ unsigned File::Read(void* dest, unsigned size)
         
         return size;
     }
-    #endif
     
     if (!handle_)
     {
@@ -321,9 +322,8 @@ unsigned File::Seek(unsigned position)
         readBufferSize_ = 0;
         return position_;
     }
-    #else
-    // Package compressed mode
-    if (readBuffer_)
+    #endif
+    if (compressed_)
     {
         // Start over from the beginning
         if (position == 0)
@@ -336,16 +336,15 @@ unsigned File::Seek(unsigned position)
         // Skip bytes
         else if (position >= position_)
         {
-            unsigned char skipBuffer[READ_BUFFER_SIZE];
+            unsigned char skipBuffer[SKIP_BUFFER_SIZE];
             while (position > position_)
-                Read(skipBuffer, Min((int)position - position_, (int)READ_BUFFER_SIZE));
+                Read(skipBuffer, Min((int)position - position_, (int)SKIP_BUFFER_SIZE));
         }
         else
             LOGERROR("Seeking backward in a compressed file is not supported");
         
         return position_;
     }
-    #endif
     
     if (!handle_)
     {
@@ -426,6 +425,7 @@ void File::Close()
     #endif
     
     readBuffer_.Reset();
+    inputBuffer_.Reset();
     
     if (handle_)
     {

Source/Engine/IO/File.h

@@ -103,6 +103,8 @@ private:
     #endif
     /// Read buffer for Android asset or compressed file loading.
     SharedArrayPtr<unsigned char> readBuffer_;
+    /// Decompression input buffer for compressed file loading.
+    SharedArrayPtr<unsigned char> inputBuffer_;
     /// Read buffer position.
     unsigned readBufferOffset_;
     /// Bytes in the current read buffer.
@@ -111,6 +113,8 @@ private:
     unsigned offset_;
     /// Content checksum.
     unsigned checksum_;
+    /// Compression flag.
+    bool compressed_;
 };
 
 }

Source/Engine/IO/PackageFile.cpp

@@ -32,7 +32,6 @@ PackageFile::PackageFile(Context* context) :
     Object(context),
     totalSize_(0),
     checksum_(0),
-    blockSize_(0),
     compressed_(false)
 {
 }
@@ -41,7 +40,6 @@ PackageFile::PackageFile(Context* context, const String& fileName) :
     Object(context),
     totalSize_(0),
     checksum_(0),
-    blockSize_(0),
     compressed_(false)
 {
     Open(fileName);
@@ -53,6 +51,14 @@ PackageFile::~PackageFile()
 
 bool PackageFile::Open(const String& fileName)
 {
+    #ifdef ANDROID
+    if (fileName.StartsWith("/apk/"))
+    {
+        LOGERROR("Package files within the apk are not supported on Android");
+        return false;
+    }
+    #endif
+    
     SharedPtr<File> file(new File(context_, fileName));
     if (!file->IsOpen())
         return false;
@@ -72,8 +78,6 @@ bool PackageFile::Open(const String& fileName)
     
     unsigned numFiles = file->ReadUInt();
     checksum_ = file->ReadUInt();
-    if (compressed_)
-        blockSize_ = file->ReadUInt();
     
     for (unsigned i = 0; i < numFiles; ++i)
     {

Source/Engine/IO/PackageFile.h

@@ -69,8 +69,6 @@ public:
     unsigned GetTotalSize() const { return totalSize_; }
     /// Return checksum of the package file contents.
     unsigned GetChecksum() const { return checksum_; }
-    /// Return compressed mode block size.
-    unsigned GetBlockSize() const { return blockSize_; }
     /// Return whether the files are compressed.
     bool IsCompressed() const { return compressed_; }
     
@@ -85,8 +83,6 @@ private:
     unsigned totalSize_;
     /// Package file checksum.
     unsigned checksum_;
-    /// Compressed mode block size.
-    unsigned blockSize_;
     /// Compressed flag.
     bool compressed_;
 };

Source/Engine/Script/IOAPI.cpp

@@ -361,6 +361,7 @@ static void RegisterPackageFile(asIScriptEngine* engine)
     engine->RegisterObjectMethod("PackageFile", "uint get_numFiles() const", asMETHOD(PackageFile, GetNumFiles), asCALL_THISCALL);
     engine->RegisterObjectMethod("PackageFile", "uint get_totalSize() const", asMETHOD(PackageFile, GetTotalSize), asCALL_THISCALL);
     engine->RegisterObjectMethod("PackageFile", "uint get_checksum() const", asMETHOD(PackageFile, GetChecksum), asCALL_THISCALL);
+    engine->RegisterObjectMethod("PackageFile", "bool compressed() const", asMETHOD(PackageFile, IsCompressed), asCALL_THISCALL);
 }
 
 void RegisterIOAPI(asIScriptEngine* engine)

Source/Extras/LuaScript/pkgs/IO/PackageFile.pkg

@@ -23,10 +23,12 @@ class PackageFile : public Object
     unsigned GetNumFiles() const;
     unsigned GetTotalSize() const;
     unsigned GetChecksum() const;
-    
+    bool IsCompressed() const;
+
     tolua_readonly tolua_property__get_set String& name;
     tolua_readonly tolua_property__get_set StringHash nameHash;
     tolua_readonly tolua_property__get_set unsigned numFiles;
     tolua_readonly tolua_property__get_set unsigned totalSize;
     tolua_readonly tolua_property__get_set unsigned checksum;
+    tolua_readonly tolua_property__is_set bool compressed;
 };

Source/Tools/PackageTool/PackageTool.cpp

@@ -176,6 +176,8 @@ void WritePackageFile(const String& fileName, const String& rootDir)
         dest.WriteUInt(entries_[i].checksum_);
     }
     
+    unsigned totalDataSize = 0;
+    
     // Write file data, calculate checksums & correct offsets
     for (unsigned i = 0; i < entries_.Size(); ++i)
     {
@@ -187,6 +189,7 @@ void WritePackageFile(const String& fileName, const String& rootDir)
             ErrorExit("Could not open file " + fileFullPath);
         
         unsigned dataSize = entries_[i].size_;
+        totalDataSize += dataSize;
         SharedArrayPtr<unsigned char> buffer(new unsigned char[dataSize]);
         
         if (srcFile.Read(&buffer[0], dataSize) != dataSize)
@@ -229,7 +232,7 @@ void WritePackageFile(const String& fileName, const String& rootDir)
                 pos += unpackedSize;
             }
             
-            PrintLine(entries_[i].name_ + " size " + String(dataSize) + " packed " + String(totalPackedBytes));
+            PrintLine(entries_[i].name_ + " in " + String(dataSize) + " out " + String(totalPackedBytes));
         }
     }
     
@@ -245,7 +248,9 @@ void WritePackageFile(const String& fileName, const String& rootDir)
         dest.WriteUInt(entries_[i].checksum_);
     }
     
-    PrintLine("Package total size " + String(dest.GetSize()));
+    PrintLine("Number of files " + String(entries_.Size()));
+    PrintLine("File data size " + String(totalDataSize));
+    PrintLine("Package size " + String(dest.GetSize()));
 }
 
 void WriteHeader(File& dest)
@@ -256,6 +261,4 @@ void WriteHeader(File& dest)
         dest.WriteFileID("ULZ4");
     dest.WriteUInt(entries_.Size());
     dest.WriteUInt(checksum_);
-    if (compress_)
-        dest.WriteUInt(blockSize_);
 }