File/PackageFile refactoring. PackageFile no longer keeps a file handle open constantly, instead handles are opened for individual files as necessary.

Documentation update.

Engine/Common/File.cpp

@@ -38,7 +38,6 @@ void scanDirectoryInternal(std::vector<std::string>& result, std::string path, c
 
 File::File(const std::string& fileName, FileMode mode) :
     mHandle(0),
-    mPackageFile(0),
     mFileName(fileName),
     mMode(mode),
     mOffset(0),
@@ -63,15 +62,20 @@ File::File(const std::string& fileName, FileMode mode) :
     fseek(mHandle, 0, SEEK_SET);
 }
 
-File::File(const std::string& fileName, File* packageFile, const PackageEntry& entry) :
-    Deserializer(entry.mSize),
+File::File(const PackageFile& package, const std::string& fileName) :
     mHandle(0),
-    mPackageFile(packageFile),
     mFileName(fileName),
-    mMode(FILE_READ),
-    mOffset(entry.mOffset),
-    mChecksum(entry.mChecksum)
+    mMode(FILE_READ)
 {
+    const PackageEntry& entry = package.getEntry(fileName);
+    mOffset = entry.mOffset;
+    mSize = entry.mSize;
+    mChecksum = entry.mChecksum;
+    
+    mHandle = fopen(getOSPath(package.getName()).c_str(), "rb");
+    if (!mHandle)
+        EXCEPTION("Could not open package file " + fileName);
+    fseek(mHandle, mOffset, SEEK_SET);
 }
 
 File::~File()
@@ -90,27 +94,15 @@ void File::read(void* dest, unsigned size)
     if (size + mPosition > mSize)
         SAFE_EXCEPTION("Attempted to read past file end");
     
-    // If reading from a package, redirect to it
-    if (mPackageFile.notNull())
-    {
-        if (mPackageFile.isExpired())
-            SAFE_EXCEPTION("Parent package file no longer exists");
-        
-        mPackageFile->seek(mPosition + mOffset);
-        mPackageFile->read(dest, size);
-    }
-    else
+    if (!mHandle)
+        SAFE_EXCEPTION("File not open");
+    
+    size_t ret = fread(dest, size, 1, mHandle);
+    if (ret != 1)
     {
-        if (!mHandle)
-            SAFE_EXCEPTION("File not open");
-        
-        size_t ret = fread(dest, size, 1, mHandle);
-        if (ret != 1)
-        {
-            // Return to the position where the read began
-            fseek(mHandle, mPosition, SEEK_SET);
-            SAFE_EXCEPTION("Error while reading from file");
-        }
+        // Return to the position where the read began
+        fseek(mHandle, mPosition + mOffset, SEEK_SET);
+        SAFE_EXCEPTION("Error while reading from file");
     }
     
     mPosition += size;
@@ -121,13 +113,10 @@ unsigned File::seek(unsigned position)
     if (position > mSize)
         position = mSize;
     
-    if ((mPackageFile.isNull()) && (position != mPosition))
-    {
-        if (!mHandle)
-            SAFE_EXCEPTION_RET("File not open", 0);
+    if (!mHandle)
+        SAFE_EXCEPTION_RET("File not open", 0);
         
-        fseek(mHandle, position, SEEK_SET);
-    }
+    fseek(mHandle, position + mOffset, SEEK_SET);
     
     mPosition = position;
     return mPosition;
@@ -147,7 +136,7 @@ void File::write(const void* data, unsigned size)
     if (fwrite(data, size, 1, mHandle) != 1)
     {
         // Return to the position where the write began
-        fseek(mHandle, mPosition, SEEK_SET);
+        fseek(mHandle, mPosition + mOffset, SEEK_SET);
         SAFE_EXCEPTION("Error while writing to file");
     }
     
@@ -172,7 +161,7 @@ void File::setName(const std::string& name)
 
 unsigned File::getChecksum()
 {
-    if ((mPackageFile) || (mChecksum))
+    if ((mOffset) || (mChecksum))
         return mChecksum;
     
     unsigned oldPos = mPosition;

Engine/Common/File.h

@@ -39,7 +39,7 @@ enum FileMode
     FILE_READWRITE
 };
 
-struct PackageEntry;
+class PackageFile;
 
 //! A file opened either through the filesystem or from within a package file
 class File : public RefCounted, public Deserializer, public Serializer
@@ -47,8 +47,8 @@ class File : public RefCounted, public Deserializer, public Serializer
 public:
     //! Construct and open the file with the specified name and open mode
     File(const std::string& fileName, FileMode mode = FILE_READ);
-    //! Construct by specifying an entry within a package file
-    File(const std::string& fileName, File* packageFile, const PackageEntry& entry);
+    //! Construct by specifying a package file source
+    File(const PackageFile& package, const std::string& fileName);
     //! Destruct. Close the file if open
     virtual ~File();
     
@@ -76,13 +76,11 @@ public:
 private:
     //! File handle
     FILE* mHandle;
-    //! Pointer to package file if opened from within a package
-    WeakPtr<File> mPackageFile;
     //! File name
     std::string mFileName;
     //! Open mode
     FileMode mMode;
-    //! Start position within a package file
+    //! Start position within a package file, 0 for regular files
     unsigned mOffset;
     //! Content checksum
     unsigned mChecksum;

Engine/Common/PackageFile.cpp

@@ -27,24 +27,27 @@
 #include "StringUtils.h"
 
 PackageFile::PackageFile(const std::string& fileName) :
-    mFile(fileName)
+    mFileName(fileName),
+    mNameHash(StringHash(fileName))
 {
-    // Calculate the filename hash and register the reverse mapping
-    mNameHash = StringHash(fileName);
-    registerHash(fileName);
+    File file(mFileName);
+    mTotalSize = file.getSize();
+    
+    // Register the reverse mapping for the hashed file name
+    registerHash(mFileName);
     
     // Read the directory
-    unsigned numFiles = mFile.readUInt();
-    mChecksum = mFile.readUInt();
+    unsigned numFiles = file.readUInt();
+    mChecksum = file.readUInt();
     
     for (unsigned i = 0; i < numFiles; ++i)
     {
-        std::string entryName = mFile.readString();
+        std::string entryName = file.readString();
         PackageEntry newEntry;
-        newEntry.mOffset = mFile.readUInt();
-        newEntry.mSize = mFile.readUInt();
-        newEntry.mChecksum = mFile.readUInt();
-        if (newEntry.mOffset + newEntry.mSize > mFile.getSize())
+        newEntry.mOffset = file.readUInt();
+        newEntry.mSize = file.readUInt();
+        newEntry.mChecksum = file.readUInt();
+        if (newEntry.mOffset + newEntry.mSize > file.getSize())
             EXCEPTION("File entry " + fileName + " outside package file");
         mEntries[toLower(entryName)] = newEntry;
     }

Engine/Common/PackageFile.h

@@ -40,7 +40,7 @@ struct PackageEntry
 class PackageFile : public RefCounted
 {
 public:
-    //! Construct and open the package file
+    //! Construct and read the package file directory
     PackageFile(const std::string& fileName);
     //! Destruct. Close the package file
     virtual ~PackageFile();
@@ -51,26 +51,26 @@ public:
     const PackageEntry& getEntry(const std::string& fileName) const;
     //! Return all file entries
     const std::map<std::string, PackageEntry>& getEntries() const { return mEntries; }
-    //! Return the File object representing the whole package file
-    File& getFile() { return mFile; }
     //! Return the package file name
-    const std::string& getName() const { return mFile.getName(); }
+    const std::string& getName() const { return mFileName; }
     //! Return hash of the package file name
     StringHash getNameHash() const { return mNameHash; }
     //! Return number of files
     unsigned getNumFiles() const { return mEntries.size(); }
     //! Return total size of the package file
-    unsigned getTotalSize() const { return mFile.getSize(); }
+    unsigned getTotalSize() const { return mTotalSize; }
     //! Return checksum of the package file contents
     unsigned getChecksum() const { return mChecksum; }
     
 private:
-    //! File object
-    File mFile;
     //! File entries
     std::map<std::string, PackageEntry> mEntries;
+    //! File name
+    std::string mFileName;
     //! Package file name hash
     StringHash mNameHash;
+    //! Package file total size
+    unsigned mTotalSize;
     //! Package file checksum
     unsigned mChecksum;
 };

Engine/Engine/Server.cpp

@@ -42,6 +42,9 @@
 
 #include "DebugNew.h"
 
+// Timeout for closing transferred package files, in milliseconds
+static const int FILE_TIMEOUT = 15 * 1000;
+
 Server::Server(Network* network) :
     mNetwork(network),
     mNetFps(30),
@@ -192,6 +195,15 @@ void Server::update(float timeStep)
         else
             ++i;
     }
+    
+    // Close file transfers that have been unused for some time
+    for (std::map<StringHash, ServerFileTransfer>::iterator i = mFileTransfers.begin(); i != mFileTransfers.end();)
+    {
+        std::map<StringHash, ServerFileTransfer>::iterator current = i;
+        ++i;
+        if (current->second.mCloseTimer.getMSec(false) > FILE_TIMEOUT)
+            mFileTransfers.erase(current);
+    }
 }
 
 bool Server::setClientScene(Connection* connection, Scene* scene)
@@ -414,9 +426,30 @@ bool Server::handleRequestFile(Connection* connection, VectorBuffer& packet)
         return true;
     }
     
-    // Check that the fragment range is not outside file
-    File& file = package->getFile();
-    unsigned fileSize = file.getSize();
+    // Open file if not already open
+    File* file = mFileTransfers[nameHash].mFile;
+    if (!file)
+    {
+        try
+        {
+            file = mFileTransfers[nameHash].mFile = new File(package->getName());
+        }
+        catch(...)
+        {
+            LOGERROR("Failed to open package file " + package->getName() + " for file transfer");
+            VectorBuffer replyPacket;
+            replyPacket.writeUByte(MSG_TRANSFERFAILED);
+            replyPacket.writeStringHash(nameHash);
+            replyPacket.writeString("Could not open file");
+            connection->sendReliable(replyPacket);
+            return true;
+        }
+    }
+    
+    mFileTransfers[nameHash].mCloseTimer.reset();
+    
+    // Check that fragment range is valid
+    unsigned fileSize = file->getSize();
     int maxFragments = (fileSize - 1) / FILE_FRAGMENT_SIZE + 1;
     if (fragmentStart + fragmentCount > maxFragments)
         fragmentCount = maxFragments - fragmentStart;
@@ -428,7 +461,7 @@ bool Server::handleRequestFile(Connection* connection, VectorBuffer& packet)
     
     // Send the fragments
     unsigned fragmentOffset = fragmentStart * FILE_FRAGMENT_SIZE;
-    file.seek(fragmentOffset);
+    file->seek(fragmentOffset);
     
     for (int i = fragmentStart; i < fragmentStart + fragmentCount; ++i)
     {
@@ -443,7 +476,7 @@ bool Server::handleRequestFile(Connection* connection, VectorBuffer& packet)
         fragmentPacket.writeStringHash(nameHash);
         fragmentPacket.writeVLE(i);
         fragmentPacket.resize(fragmentPacket.getPosition() + fragmentSize);
-        file.read(fragmentPacket.getModifiableData() + fragmentPacket.getPosition(), fragmentSize);
+        file->read(fragmentPacket.getModifiableData() + fragmentPacket.getPosition(), fragmentSize);
         connection->sendReliable(fragmentPacket);
     }
     

Engine/Engine/Server.h

@@ -25,6 +25,7 @@
 #define ENGINE_SERVER_H
 
 #include "EventListener.h"
+#include "Timer.h"
 #include "SharedPtr.h"
 
 #include <vector>
@@ -34,6 +35,13 @@ class Network;
 class Scene;
 class VectorBuffer;
 
+//! Server-side ongoing download, with a timer for closing the file if unused
+struct ServerFileTransfer
+{
+    SharedPtr<File> mFile;
+    Timer mCloseTimer;
+};
+
 //! Provides scenes over the network
 class Server : public RefCounted, public EventListener
 {
@@ -127,6 +135,8 @@ private:
     std::vector<SharedPtr<Scene> > mScenes;
     //! Client connections
     std::vector<SharedPtr<Connection> > mConnections;
+    //! Ongoing file downloads
+    std::map<StringHash, ServerFileTransfer> mFileTransfers;
     //! Network updates per second
     int mNetFps;
     //! Network update time accumulator

Engine/Resource/ResourceCache.cpp

@@ -246,10 +246,7 @@ SharedPtr<File> ResourceCache::getFile(const std::string& name)
     for (unsigned i = 0; i < mPackages.size(); ++i)
     {
         if (mPackages[i]->exists(name))
-        {
-            const PackageEntry& entry = mPackages[i]->getEntry(name);
-            return SharedPtr<File>(new File(name, &mPackages[i]->getFile(), entry));
-        }
+            return SharedPtr<File>(new File(*mPackages[i], name));
     }
     
     // Then the filesystem