optimizations to vfs.readFile(), particular for mounted multifiles

panda/src/express/multifile.cxx

@@ -21,6 +21,7 @@
 #include "zStream.h"
 #include "encryptStream.h"
 #include "virtualFileSystem.h"
+#include "virtualFile.h"
 
 #include <algorithm>
 #include <time.h>
@@ -1996,14 +1997,72 @@ read_subfile(int index, pvector<unsigned char> &result) {
   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
   result.clear();
 
-  istream *in = open_read_subfile(index);
-  if (in == (istream *)NULL) {
+  // Now look up the particular Subfile we are reading.
+  nassertr(is_read_valid(), NULL);
+  nassertr(index >= 0 && index < (int)_subfiles.size(), NULL);
+  Subfile *subfile = _subfiles[index];
+
+  if (subfile->_source != (istream *)NULL ||
+      !subfile->_source_filename.empty()) {
+    // The subfile has not yet been copied into the physical
+    // Multifile.  Force a flush operation to incorporate it.
+    flush();
+
+    // That shouldn't change the subfile index or delete the subfile
+    // pointer.
+    nassertr(subfile == _subfiles[index], NULL);
+  }
+
+  result.reserve(subfile->_uncompressed_length);
+
+  bool success = true;
+  if (subfile->_flags & (SF_encrypted | SF_compressed)) {
+    // If the subfile is encrypted or compressed, we can't read it
+    // directly.  Fall back to the generic implementation.
+    istream *in = open_read_subfile(index);
+    if (in == (istream *)NULL) {
+      return false;
+    }
+
+    success = VirtualFile::simple_read_file(in, result);
+    close_read_subfile(in);
+
+  } else {
+    // But if the subfile is just a plain file, we can just read the
+    // data directly from the Multifile, without paying the cost of an
+    // ISubStream.
+    static const size_t buffer_size = 4096;
+    char buffer[buffer_size];
+
+    streamsize pos = _offset + subfile->_data_start;
+    size_t max_bytes = subfile->_data_length;
+    streamsize count = 0;
+    bool eof = true;
+
+    streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
+    _read->seek_read(pos, buffer, num_bytes, count, eof);
+    while (count != 0) {
+      thread_consider_yield();
+      nassertr(count <= max_bytes, false);
+      result.insert(result.end(), buffer, buffer + (size_t)count);
+      max_bytes -= (size_t)count;
+      pos += count;
+
+      num_bytes = (streamsize)min(buffer_size, max_bytes);
+      _read->seek_read(pos, buffer, num_bytes, count, eof);
+    }
+
+    success = !eof;
+  }
+
+  if (!success) {
+    ostringstream message;
+    message << "I/O error reading from " << get_multifile_name() << " at "
+            << get_subfile_name(index);
+    nassert_raise(message.str());
     return false;
   }
 
-  bool success = read_to_pvector(result, *in);
-  close_read_subfile(in);
-  nassertr(success, false);
   return true;
 }
 
@@ -2176,31 +2235,6 @@ standardize_subfile_name(const string &subfile_name) const {
   }
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: Multifile::read_to_pvector
-//       Access: Private, Static
-//  Description: Helper function to read the entire contents of the
-//               indicated stream from the current position, and
-//               append it onto the indicated pvector.  Returns true
-//               on success, false on failure.
-////////////////////////////////////////////////////////////////////
-bool Multifile::
-read_to_pvector(pvector<unsigned char> &result, istream &stream) {
-  static const size_t buffer_size = 4096;
-  char buffer[buffer_size];
-
-  stream.read(buffer, buffer_size);
-  size_t count = stream.gcount();
-  while (count != 0) {
-    result.insert(result.end(), buffer, buffer + count);
-    stream.read(buffer, buffer_size);
-    count = stream.gcount();
-  }
-
-  bool failed = stream.fail() && !stream.eof();
-  return !failed;
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: Multifile::clear_subfiles
 //       Access: Private
@@ -2494,7 +2528,7 @@ check_signatures() {
 
     // Read the remaining buffer of certificate data.
     pvector<unsigned char> buffer;
-    bool success = read_to_pvector(buffer, *stream);
+    bool success = VirtualFile::simple_read_file(stream, buffer);
     nassertv(success);
     close_read_subfile(stream);
 

panda/src/express/multifile.h

@@ -205,7 +205,6 @@ private:
   void add_new_subfile(Subfile *subfile, int compression_level);
   istream *open_read_subfile(Subfile *subfile);
   string standardize_subfile_name(const string &subfile_name) const;
-  static bool read_to_pvector(pvector<unsigned char> &result, istream &stream);
 
   void clear_subfiles();
   bool read_index();

panda/src/express/virtualFile.cxx

@@ -278,35 +278,18 @@ read_file(string &result, bool auto_unwrap) const {
 
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFile::read_file
-//       Access: Public
+//       Access: Public, Virtual
 //  Description: Fills up the indicated pvector with the contents of
 //               the file, if it is a regular file.  Returns true on
 //               success, false otherwise.
 ////////////////////////////////////////////////////////////////////
 bool VirtualFile::
 read_file(pvector<unsigned char> &result, bool auto_unwrap) const {
-  result.clear();
-
-  istream *in = open_read_file(auto_unwrap);
-  if (in == (istream *)NULL) {
-    express_cat.info()
-      << "Unable to read " << get_filename() << "\n";
-    return false;
-  }
-
-  bool okflag = read_file(in, result);
-
-  close_read_file(in);
-
-  if (!okflag) {
-    express_cat.info()
-      << "Error while reading " << get_filename() << "\n";
-  }
-  return okflag;
+  return false;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: VirtualFile::read_file
+//     Function: VirtualFile::simple_read_file
 //       Access: Public, Static
 //  Description: Fills up the indicated pvector with the contents of
 //               the just-opened file.  Returns true on success, false
@@ -314,10 +297,8 @@ read_file(pvector<unsigned char> &result, bool auto_unwrap) const {
 //               data read from the file will be appended onto it.
 ////////////////////////////////////////////////////////////////////
 bool VirtualFile::
-read_file(istream *in, pvector<unsigned char> &result) {
-  pvector<char> result_vec;
-
-  static const size_t buffer_size = 1024;
+simple_read_file(istream *in, pvector<unsigned char> &result) {
+  static const size_t buffer_size = 4096;
   char buffer[buffer_size];
 
   in->read(buffer, buffer_size);
@@ -333,14 +314,14 @@ read_file(istream *in, pvector<unsigned char> &result) {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: VirtualFile::read_file
-//       Access: Public, Static
-//  Description: As in read_file() with two parameters, above, but
-//               only reads up to max_bytes bytes from the file.
+//     Function: VirtualFile::simple_read_file
+//       Access: Public
+//  Description: As in simple_read_file() with two parameters, above,
+//               but only reads up to max_bytes bytes from the file.
 ////////////////////////////////////////////////////////////////////
 bool VirtualFile::
-read_file(istream *in, pvector<unsigned char> &result, size_t max_bytes) {
-  static const size_t buffer_size = 1024;
+simple_read_file(istream *in, pvector<unsigned char> &result, size_t max_bytes) {
+  static const size_t buffer_size = 4096;
   char buffer[buffer_size];
 
   in->read(buffer, min(buffer_size, max_bytes));

panda/src/express/virtualFile.h

@@ -63,9 +63,10 @@ PUBLISHED:
 public:
   INLINE void set_original_filename(const Filename &filename);
   bool read_file(string &result, bool auto_unwrap) const;
-  bool read_file(pvector<unsigned char> &result, bool auto_unwrap) const;
-  static bool read_file(istream *stream, pvector<unsigned char> &result);
-  static bool read_file(istream *stream, pvector<unsigned char> &result, size_t max_bytes);
+  virtual bool read_file(pvector<unsigned char> &result, bool auto_unwrap) const;
+
+  static bool simple_read_file(istream *stream, pvector<unsigned char> &result);
+  static bool simple_read_file(istream *stream, pvector<unsigned char> &result, size_t max_bytes);
 
 
 protected:

panda/src/express/virtualFileMount.cxx

@@ -14,6 +14,7 @@
 
 #include "virtualFileMount.h"
 #include "virtualFileSimple.h"
+#include "zStream.h"
 
 TypeHandle VirtualFileMount::_type_handle;
 
@@ -53,6 +54,67 @@ make_virtual_file(const Filename &local_filename,
   return file.p();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMount::read_file
+//       Access: Public, Virtual
+//  Description: Fills up the indicated pvector with the contents of
+//               the file, if it is a regular file.  Returns true on
+//               success, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMount::
+read_file(const Filename &file, bool do_uncompress,
+          pvector<unsigned char> &result) const {
+  result.clear();
+
+  istream *in = open_read_file(file, do_uncompress);
+  if (in == (istream *)NULL) {
+    express_cat.info()
+      << "Unable to read " << file << "\n";
+    return false;
+  }
+
+  off_t file_size = get_file_size(file, in);
+  if (file_size != 0) {
+    result.reserve((size_t)file_size);
+  }
+
+  bool okflag = VirtualFile::simple_read_file(in, result);
+
+  close_read_file(in);
+
+  if (!okflag) {
+    express_cat.info()
+      << "Error while reading " << file << "\n";
+  }
+  return okflag;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMount::open_read_file
+//       Access: Published, Virtual
+//  Description: Opens the file for reading.  Returns a newly
+//               allocated istream on success (which you should
+//               eventually delete when you are done reading).
+//               Returns NULL on failure.
+//
+//               If do_uncompress is true, the file is also
+//               decompressed on-the-fly using zlib.
+////////////////////////////////////////////////////////////////////
+istream *VirtualFileMount::
+open_read_file(const Filename &file, bool do_uncompress) const {
+  istream *result = open_read_file(file);
+
+#ifdef HAVE_ZLIB
+  if (result != (istream *)NULL && do_uncompress) {
+    // We have to slip in a layer to decompress the file on the fly.
+    IDecompressStream *wrapper = new IDecompressStream(result, true);
+    result = wrapper;
+  }
+#endif  // HAVE_ZLIB
+
+  return result;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileMount::close_read_file
 //       Access: Public

panda/src/express/virtualFileMount.h

@@ -49,7 +49,11 @@ public:
   virtual bool is_directory(const Filename &file) const=0;
   virtual bool is_regular_file(const Filename &file) const=0;
 
+  virtual bool read_file(const Filename &file, bool do_uncompress,
+                         pvector<unsigned char> &result) const;
+
   virtual istream *open_read_file(const Filename &file) const=0;
+  istream *open_read_file(const Filename &file, bool do_uncompress) const;
   void close_read_file(istream *stream) const;
   virtual off_t get_file_size(const Filename &file, istream *stream) const=0;
   virtual off_t get_file_size(const Filename &file) const=0;

panda/src/express/virtualFileMountMultifile.cxx

@@ -63,6 +63,36 @@ is_regular_file(const Filename &file) const {
   return (_multifile->find_subfile(file) >= 0);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountMultifile::read_file
+//       Access: Public, Virtual
+//  Description: Fills up the indicated pvector with the contents of
+//               the file, if it is a regular file.  Returns true on
+//               success, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMountMultifile::
+read_file(const Filename &file, bool do_uncompress,
+          pvector<unsigned char> &result) const {
+  if (do_uncompress) {
+    // If the file is to be decompressed, we'd better just use the
+    // higher-level implementation, which includes support for
+    // on-the-fly decompression.
+    return VirtualFileMount::read_file(file, do_uncompress, result);
+  }
+
+  // But if we're just reading a straight file, let the Multifile do
+  // the reading, which avoids a few levels of buffer copies.
+
+  int subfile_index = _multifile->find_subfile(file);
+  if (subfile_index < 0) {
+    express_cat.info()
+      << "Unable to read " << file << "\n";
+    return false;
+  }
+
+  return _multifile->read_subfile(subfile_index, result);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileMountMultifile::open_read_file
 //       Access: Public, Virtual

panda/src/express/virtualFileMountMultifile.h

@@ -38,6 +38,9 @@ public:
   virtual bool is_directory(const Filename &file) const;
   virtual bool is_regular_file(const Filename &file) const;
 
+  virtual bool read_file(const Filename &file, bool do_uncompress,
+                         pvector<unsigned char> &result) const;
+
   virtual istream *open_read_file(const Filename &file) const;
   virtual off_t get_file_size(const Filename &file, istream *stream) const;
   virtual off_t get_file_size(const Filename &file) const;

panda/src/express/virtualFileSimple.cxx

@@ -15,7 +15,6 @@
 #include "virtualFileSimple.h"
 #include "virtualFileMount.h"
 #include "virtualFileList.h"
-#include "zStream.h"
 
 TypeHandle VirtualFileSimple::_type_handle;
 
@@ -106,24 +105,15 @@ istream *VirtualFileSimple::
 open_read_file(bool auto_unwrap) const {
 
   // Will we be automatically unwrapping a .pz file?
-  bool do_unwrap = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz"));
+  bool do_uncompress = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz"));
 
   Filename local_filename(_local_filename);
-  if (do_unwrap) {
+  if (do_uncompress) {
     // .pz files are always binary, of course.
     local_filename.set_binary();
   }
 
-  istream *result = _mount->open_read_file(local_filename);
-#ifdef HAVE_ZLIB
-  if (result != (istream *)NULL && do_unwrap) {
-    // We have to slip in a layer to decompress the file on the fly.
-    IDecompressStream *wrapper = new IDecompressStream(result, true);
-    result = wrapper;
-  }
-#endif  // HAVE_ZLIB
-
-  return result;
+  return _mount->open_read_file(local_filename, do_uncompress);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -170,6 +160,28 @@ get_timestamp() const {
   return _mount->get_timestamp(_local_filename);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSimple::read_file
+//       Access: Public, Virtual
+//  Description: Fills up the indicated pvector with the contents of
+//               the file, if it is a regular file.  Returns true on
+//               success, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileSimple::
+read_file(pvector<unsigned char> &result, bool auto_unwrap) const {
+
+  // Will we be automatically unwrapping a .pz file?
+  bool do_uncompress = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz"));
+
+  Filename local_filename(_local_filename);
+  if (do_uncompress) {
+    // .pz files are always binary, of course.
+    local_filename.set_binary();
+  }
+
+  return _mount->read_file(local_filename, do_uncompress, result);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSimple::scan_local_directory
 //       Access: Protected, Virtual

panda/src/express/virtualFileSimple.h

@@ -47,6 +47,9 @@ PUBLISHED:
   virtual off_t get_file_size() const;
   virtual time_t get_timestamp() const;
 
+public:
+  virtual bool read_file(pvector<unsigned char> &result, bool auto_unwrap) const;
+
 protected:
   virtual bool scan_local_directory(VirtualFileList *file_list, 
                                     const ov_set<string> &mount_points) const;

panda/src/express/virtualFileSystem.cxx

@@ -765,6 +765,32 @@ get_global_ptr() {
   return _global_ptr;
 }
 
+#ifdef HAVE_PYTHON
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSystem::__py__read_file
+//       Access: Published
+//  Description: Convenience function; returns the entire contents of
+//               the indicated file as a string.
+//
+//               This variant on read_file() is implemented directly
+//               for Python, as a small optimization, to avoid the
+//               double-construction of a string object that would be
+//               otherwise required for the return value.
+////////////////////////////////////////////////////////////////////
+PyObject *VirtualFileSystem::
+__py__read_file(const Filename &filename, bool auto_unwrap) const {
+  pvector<unsigned char> pv;
+  bool okflag = read_file(filename, pv, auto_unwrap);
+  nassertr(okflag, NULL);
+
+  if (pv.empty()) {
+    return PyString_FromStringAndSize("", 0);
+  } else {
+    return PyString_FromStringAndSize((const char *)&pv[0], pv.size());
+  }
+}
+#endif  // HAVE_PYTHON
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSystem::open_read_file
 //       Access: Published

panda/src/express/virtualFileSystem.h

@@ -91,7 +91,11 @@ PUBLISHED:
 
   static VirtualFileSystem *get_global_ptr();
 
+#ifdef HAVE_PYTHON
+  BLOCKING PyObject *__py__read_file(const Filename &filename, bool auto_unwrap) const;
+#endif  // HAVE_PYTHON
   BLOCKING INLINE string read_file(const Filename &filename, bool auto_unwrap) const;
+
   BLOCKING istream *open_read_file(const Filename &filename, bool auto_unwrap) const;
   BLOCKING static void close_read_file(istream *stream);