more fixes and optimizations to subfile patcher

panda/src/downloadertools/multify.cxx

@@ -32,8 +32,9 @@
 bool create = false;           // -c
 bool append = false;           // -r
 bool update = false;           // -u
-bool tlist = false;             // -t
+bool tlist = false;            // -t
 bool extract = false;          // -x
+bool kill = false;             // -k
 bool verbose = false;          // -v
 bool compress_flag = false;         // -z
 int default_compression_level = 6;
@@ -138,9 +139,13 @@ help() {
     "      Extract the contents of an existing Multifile.  The Subfiles named on\n"
     "      the command line, or all Subfiles if nothing is named, are extracted\n"
     "      into the current directory or into whichever directory is specified\n"
-    "      with -C.\n\n\n"
+    "      with -C.\n\n"
 
-    
+    "  -k\n"
+    "      Delete (kill) the named Subfiles from the Multifile.  The Multifile\n"
+    "      will be repacked after completion.\n\n"
+
+    "\n"
     "  You must always specify the following switch:\n\n"
 
     "  -f <multifile_name>\n"
@@ -437,6 +442,41 @@ extract_files(int argc, char *argv[]) {
   return true;
 }
 
+bool
+kill_files(int argc, char *argv[]) {
+  if (!multifile_name.exists()) {
+    cerr << multifile_name << " not found.\n";
+    return false;
+  }
+  PT(Multifile) multifile = new Multifile;
+  if (!multifile->open_read_write(multifile_name)) {
+    cerr << "Unable to open " << multifile_name << " for read/write.\n";
+    return false;
+  }
+
+  int i = 0;
+  while (i < multifile->get_num_subfiles()) {
+    string subfile_name = multifile->get_subfile_name(i);
+    if (is_named(subfile_name, argc, argv)) {
+      Filename filename = subfile_name;
+
+      if (verbose) {
+        cout << filename << "\n";
+      }
+      multifile->remove_subfile(i);
+    } else {
+      ++i;
+    }
+  }
+
+  if (!multifile->repack()) {
+    cerr << "Failed to write " << multifile_name << ".\n";
+    return false;
+  }
+
+  return true;
+}
+
 const char *
 format_timestamp(bool record_timestamp, time_t timestamp) {
   static const size_t buffer_size = 512;
@@ -582,7 +622,7 @@ main(int argc, char *argv[]) {
 
   extern char *optarg;
   extern int optind;
-  static const char *optflags = "crutxvz123456789Z:T:f:OC:ep:F:h";
+  static const char *optflags = "crutxkvz123456789Z:T:f:OC:ep:F:h";
   int flag = getopt(argc, argv, optflags);
   Filename rel_path;
   while (flag != EOF) {
@@ -602,6 +642,9 @@ main(int argc, char *argv[]) {
     case 'x':
       extract = true;
       break;
+    case 'k':
+      kill = true;
+      break;
     case 'v':
       verbose = true;
       break;
@@ -711,8 +754,8 @@ main(int argc, char *argv[]) {
   argv += (optind - 1);
 
   // We should have exactly one of these options.
-  if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) != 1) {
-    cerr << "Exactly one of -c, -r, -u, -t, -x must be specified.\n";
+  if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) + (kill?1:0) != 1) {
+    cerr << "Exactly one of -c, -r, -u, -t, -x, -k must be specified.\n";
     usage();
     return 1;
   }
@@ -734,6 +777,11 @@ main(int argc, char *argv[]) {
       cerr << "Warning: -T ignored on extract.\n";
     }
     okflag = extract_files(argc, argv);
+  } else if (kill) {
+    if (got_record_timestamp_flag) {
+      cerr << "Warning: -T ignored on kill.\n";
+    }
+    okflag = kill_files(argc, argv);
   } else { // list
     if (got_record_timestamp_flag) {
       cerr << "Warning: -T ignored on list.\n";

panda/src/express/hashVal.cxx

@@ -211,6 +211,25 @@ hash_file(const Filename &filename) {
     return false;
   }
 
+  bool result = hash_stream(*istr);
+  vfs->close_read_file(istr);
+
+  return result;
+}
+#endif  // HAVE_OPENSSL
+
+#ifdef HAVE_OPENSSL
+////////////////////////////////////////////////////////////////////
+//     Function: HashVal::hash_stream
+//       Access: Published
+//  Description: Generates the hash value from the indicated file.
+//               Returns true on success, false if the file cannot be
+//               read.  This method is only defined if we have the
+//               OpenSSL library (which provides md5 functionality)
+//               available.
+////////////////////////////////////////////////////////////////////
+bool HashVal::
+hash_stream(istream &stream) {
   unsigned char md[16];
 
   MD5_CTX ctx;
@@ -219,15 +238,21 @@ hash_file(const Filename &filename) {
   static const int buffer_size = 1024;
   char buffer[buffer_size];
 
-  istr->read(buffer, buffer_size);
-  size_t count = istr->gcount();
+  // Seek the stream to the beginning in case it wasn't there already.
+  stream.seekg(0, ios::beg);
+
+  stream.read(buffer, buffer_size);
+  size_t count = stream.gcount();
   while (count != 0) {
     MD5_Update(&ctx, buffer, count);
-    istr->read(buffer, buffer_size);
-    count = istr->gcount();
+    stream.read(buffer, buffer_size);
+    count = stream.gcount();
   }
+  
+  // Clear the fail bit so the caller can still read the stream (if it
+  // wants to).
+  stream.clear();
 
-  vfs->close_read_file(istr);
   MD5_Final(md, &ctx);
 
   // Store the individual bytes as big-endian ints, from historical
@@ -239,7 +264,10 @@ hash_file(const Filename &filename) {
 
   return true;
 }
+#endif  // HAVE_OPENSSL
 
+
+#ifdef HAVE_OPENSSL
 ////////////////////////////////////////////////////////////////////
 //     Function: HashVal::hash_buffer
 //       Access: Published

panda/src/express/hashVal.h

@@ -71,6 +71,7 @@ PUBLISHED:
 
 #ifdef HAVE_OPENSSL
   bool hash_file(const Filename &filename);
+  bool hash_stream(istream &stream);
   INLINE void hash_ramfile(const Ramfile &ramfile);
   INLINE void hash_string(const string &data);
   void hash_buffer(const char *buffer, int length);

panda/src/express/multifile.cxx

@@ -1161,6 +1161,7 @@ open_read(istream *multifile_stream) {
   _timestamp = time(NULL);
   _timestamp_dirty = true;
   _read = multifile_stream;
+  _read->seekg(0, ios::beg);
   return read_index();
 }
 
@@ -1180,6 +1181,7 @@ open_write(ostream *multifile_stream) {
   _timestamp = time(NULL);
   _timestamp_dirty = true;
   _write = multifile_stream;
+  _write->seekp(0, ios::beg);
   return true;
 }
 
@@ -1201,6 +1203,7 @@ open_read_write(iostream *multifile_stream) {
   _timestamp_dirty = true;
   _read = multifile_stream;
   _write = multifile_stream;
+  _write->seekp(0, ios::beg);
 
   // Check whether the read stream is empty.
   _read->seekg(0, ios::end);
@@ -1211,7 +1214,7 @@ open_read_write(iostream *multifile_stream) {
 
   // The read stream is not empty, so we'd better have a valid
   // Multifile.
-  _read->seekg(0);
+  _read->seekg(0, ios::beg);
   return read_index();
 }
 

panda/src/express/patchfile.cxx

@@ -26,6 +26,7 @@
 #include "streamReader.h"
 #include "streamWriter.h"
 #include "multifile.h"
+#include "hashVal.h"
 
 #include <stdio.h> // for tempnam
 
@@ -109,6 +110,7 @@ const PN_uint32 Patchfile::_HASH_MASK = (PN_uint32(1) << Patchfile::_HASH_BITS)
 ////////////////////////////////////////////////////////////////////
 Patchfile::
 Patchfile() {
+  _hash_table = NULL;
   PT(Buffer) buffer = new Buffer(patchfile_buffer_size);
   init(buffer);
 }
@@ -120,6 +122,7 @@ Patchfile() {
 ////////////////////////////////////////////////////////////////////
 Patchfile::
 Patchfile(PT(Buffer) buffer) {
+  _hash_table = NULL;
   init(buffer);
 }
 
@@ -146,6 +149,10 @@ init(PT(Buffer) buffer) {
 ////////////////////////////////////////////////////////////////////
 Patchfile::
 ~Patchfile() {
+  if (_hash_table != (PN_uint32 *)NULL) {
+    delete[] _hash_table;
+  }
+
   if (true == _initiated)
     cleanup();
 }
@@ -299,7 +306,7 @@ run() {
     _total_bytes_processed += (int)ADD_length;
 
     // if there are bytes to add, read them from patch file and write them to output
-    if (express_cat.is_spam()) {
+    if (express_cat.is_spam() && ADD_length != 0) {
       express_cat.spam()
         << "ADD: " << ADD_length << " (to " 
         << _write_stream.tellp() << ")" << endl;
@@ -762,14 +769,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_pos, PN_uint16 &copy_lengt
 //  Description:
 ////////////////////////////////////////////////////////////////////
 void Patchfile::
-emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
-         PN_uint32 ADD_pos) {
-
+emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer) {
   nassertv(length == (PN_uint16)length); //we only write a uint16
 
   if (express_cat.is_spam()) {
     express_cat.spam()
-      << "ADD: " << length << " (to " << ADD_pos << ")" << endl;
+      << "ADD: " << length << " (to " << _add_pos << ")" << endl;
   }
 
   // write ADD length
@@ -780,6 +785,8 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
   if (length > 0) {
     patch_writer.append_data(buffer, (PN_uint16)length);
   }
+
+  _add_pos += length;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -787,29 +794,28 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
 //       Access: Private
 //  Description:
 ////////////////////////////////////////////////////////////////////
-PN_uint32 Patchfile::
-emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos,
-          PN_uint32 last_copy_pos, PN_uint32 ADD_pos) {
-
-  nassertr(length == (PN_uint16)length, last_copy_pos); //we only write a uint16
+void Patchfile::
+emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 copy_pos) {
+  nassertv(length == (PN_uint16)length); //we only write a uint16
 
-  PN_int32 offset = (int)COPY_pos - (int)last_copy_pos;
+  PN_int32 offset = (int)copy_pos - (int)_last_copy_pos;
   if (express_cat.is_spam()) {
     express_cat.spam()
       << "COPY: " << length << " bytes from offset " << offset
-      << " (from " << COPY_pos << " to " << ADD_pos << ")" << endl;
+      << " (from " << copy_pos << " to " << _add_pos << ")" << endl;
   }
 
   // write COPY length
   StreamWriter patch_writer(write_stream);
   patch_writer.add_uint16((PN_uint16)length);
 
-  if((PN_uint16)length > 0) {
+  if ((PN_uint16)length != 0) {
     // write COPY offset
     patch_writer.add_int32(offset);
+    _last_copy_pos = copy_pos + length;
   }
 
-  return COPY_pos + length;
+  _add_pos += length;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -819,42 +825,91 @@ emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos,
 //               pair as needed to work around the 16-bit chunk size
 //               limit.
 ////////////////////////////////////////////////////////////////////
-PN_uint32 Patchfile::
+void Patchfile::
 emit_add_and_copy(ostream &write_stream, 
                   PN_uint32 add_length, const char *add_buffer,
-                  PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos,
-                  PN_uint32 add_pos) {
-  while (add_length > 65535) {
+                  PN_uint32 copy_length, PN_uint32 copy_pos) {
+  if (add_length == 0 && copy_length == 0) {
+    // Don't accidentally emit a termination code.
+    return;
+  }
+
+  static const PN_uint16 max_write = 65535;
+  while (add_length > max_write) {
     // Overflow.  This chunk is too large to fit into a single
     // ADD block, so we have to write it as multiple ADDs.
-    static const PN_uint16 max_write = 65535;
-    emit_ADD(write_stream, max_write, add_buffer, add_pos);
-    add_pos += max_write;
+    emit_ADD(write_stream, max_write, add_buffer);
     add_buffer += max_write;
     add_length -= max_write;
-    emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, add_pos);
+    emit_COPY(write_stream, 0, 0);
   }
 
-  emit_ADD(write_stream, add_length, add_buffer, add_pos);
+  emit_ADD(write_stream, add_length, add_buffer);
 
-  while (copy_length > 65535) {
+  while (copy_length > max_write) {
     // Overflow.
-    static const PN_uint16 max_write = 65535;
-    last_copy_pos =
-      emit_COPY(write_stream, max_write, copy_pos, last_copy_pos, add_pos);
+    emit_COPY(write_stream, max_write, copy_pos);
     copy_pos += max_write;
-    add_pos += max_write;
     copy_length -= max_write;
-    emit_ADD(write_stream, 0, NULL, add_pos);
+    emit_ADD(write_stream, 0, NULL);
   }
 
-  last_copy_pos =
-    emit_COPY(write_stream, copy_length, copy_pos, last_copy_pos, add_pos);
-
-  return last_copy_pos;
+  emit_COPY(write_stream, copy_length, copy_pos);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: Patchfile::cache_add_and_copy
+//       Access: Private
+//  Description: Potentially emits one or more add/copy pairs.  The
+//               current state is saved, so as to minimize wasted
+//               emits from consecutive adds or copies.
+////////////////////////////////////////////////////////////////////
+void Patchfile::
+cache_add_and_copy(ostream &write_stream, 
+                   PN_uint32 add_length, const char *add_buffer,
+                   PN_uint32 copy_length, PN_uint32 copy_pos) {
+  if (add_length != 0) {
+    if (_cache_copy_length != 0) {
+      // Have to flush.
+      cache_flush(write_stream);
+    }
+    // Add the string to the current cache.
+    _cache_add_data += string(add_buffer, add_length);
+  }
+
+  if (copy_length != 0) {
+    if (_cache_copy_length == 0) {
+      // Start a new copy phase.
+      _cache_copy_start = copy_pos;
+      _cache_copy_length = copy_length;
+
+    } else if (_cache_copy_start + _cache_copy_length == copy_pos) {
+      // We can just tack on the copy to what we've already got.
+      _cache_copy_length += copy_length;
 
+    } else {
+      // It's a discontinuous copy.  We have to flush.
+      cache_flush(write_stream);
+      _cache_copy_start = copy_pos;
+      _cache_copy_length = copy_length;
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Patchfile::cache_flush
+//       Access: Private
+//  Description: Closes any copy or add phases that are still open
+//               after a previous call to cache_add_and_copy().
+////////////////////////////////////////////////////////////////////
+void Patchfile::
+cache_flush(ostream &write_stream) {
+  emit_add_and_copy(write_stream, 
+                    _cache_add_data.size(), _cache_add_data.data(),
+                    _cache_copy_length, _cache_copy_start);
+  _cache_add_data = string();
+  _cache_copy_length = 0;
+}
 
 
 ////////////////////////////////////////////////////////////////////
@@ -865,8 +920,7 @@ emit_add_and_copy(ostream &write_stream,
 ////////////////////////////////////////////////////////////////////
 void Patchfile::
 write_header(ostream &write_stream, 
-             char *buffer_orig, PN_uint32 source_file_length,
-             char *buffer_new, PN_uint32 result_file_length) {
+             istream &stream_orig, istream &stream_new) {
   // prepare to write the patch file header
 
   START_PROFILE(writeHeader);
@@ -875,19 +929,33 @@ write_header(ostream &write_stream,
   StreamWriter patch_writer(write_stream);
   patch_writer.add_uint32(_magic_number);
   patch_writer.add_uint16(_current_version);
-  patch_writer.add_uint32(source_file_length);
-  {
-    // calc MD5 of original file
-    _MD5_ofSource.hash_buffer(buffer_orig, source_file_length);
-    // add it to the header
-    _MD5_ofSource.write_stream(patch_writer);
+
+  stream_orig.seekg(0, ios::end);
+  streampos source_file_length = stream_orig.tellg();
+  patch_writer.add_uint32((PN_uint32)source_file_length);
+
+  // calc MD5 of original file
+  _MD5_ofSource.hash_stream(stream_orig);
+  // add it to the header
+  _MD5_ofSource.write_stream(patch_writer);
+
+  if (express_cat.is_debug()) {
+    express_cat.debug()
+      << "Orig: " << _MD5_ofSource << "\n";
   }
-  patch_writer.add_uint32(result_file_length);
-  {
-    // calc MD5 of resultant patched file
-    _MD5_ofResult.hash_buffer(buffer_new, result_file_length);
-    // add it to the header
-    _MD5_ofResult.write_stream(patch_writer);
+
+  stream_new.seekg(0, ios::end);
+  streampos result_file_length = stream_new.tellg();
+  patch_writer.add_uint32((PN_uint32)result_file_length);
+
+  // calc MD5 of resultant patched file
+  _MD5_ofResult.hash_stream(stream_new);
+  // add it to the header
+  _MD5_ofResult.write_stream(patch_writer);
+
+  if (express_cat.is_debug()) {
+    express_cat.debug()
+      << " New: " << _MD5_ofResult << "\n";
   }
 
   END_PROFILE(writeHeader, "writing patch file header");
@@ -900,11 +968,11 @@ write_header(ostream &write_stream,
 //               Writes the patchfile terminator.
 ////////////////////////////////////////////////////////////////////
 void Patchfile::
-write_terminator(ostream &write_stream, PN_uint32 result_file_length,
-                 PN_uint32 last_copy_pos) {
+write_terminator(ostream &write_stream) {
+  cache_flush(write_stream);
   // write terminator (null ADD, null COPY)
-  emit_ADD(write_stream, 0, NULL, result_file_length);
-  emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, result_file_length);
+  emit_ADD(write_stream, 0, NULL);
+  emit_COPY(write_stream, 0, 0);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -914,23 +982,47 @@ write_terminator(ostream &write_stream, PN_uint32 result_file_length,
 //               Computes the patches for the entire file (if it is
 //               not a multifile) or for a single subfile (if it is)
 //
-//               Returns last_copy_pos, the last byte position from
-//               which we copied from the original file.
+//               Returns true if successful, false on error.
 ////////////////////////////////////////////////////////////////////
-PN_uint32 Patchfile::
-compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
-                PN_uint32 copy_offset,
-                char *buffer_orig, PN_uint32 source_file_length,
-                char *buffer_new, PN_uint32 result_file_length) {
-  START_PROFILE(allocTables);
+bool Patchfile::
+compute_patches(ostream &write_stream, PN_uint32 copy_offset,
+                istream &stream_orig, istream &stream_new) {
+  // read in original file
+  stream_orig.seekg(0, ios::end);
+  nassertr(stream_orig, false);
+  PN_uint32 source_file_length = stream_orig.tellg();
+  if (express_cat.is_debug()) {
+    express_cat.debug()
+      << "Allocating " << source_file_length << " bytes to read orig\n";
+  }
 
-  // allocate hash/link tables
+  char *buffer_orig = new char[source_file_length];
+  stream_orig.seekg(0, ios::beg);
+  stream_orig.read(buffer_orig, source_file_length);
+
+  // read in new file
+  stream_new.seekg(0, ios::end);
+  PN_uint32 result_file_length = stream_new.tellg();
+  nassertr(stream_new, false);
   if (express_cat.is_debug()) {
     express_cat.debug()
-      << "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n";
+      << "Allocating " << result_file_length << " bytes to read new\n";
   }
 
-  PN_uint32* hash_table = new PN_uint32[_HASHTABLESIZE];
+  char *buffer_new = new char[result_file_length];
+  stream_new.seekg(0, ios::beg);
+  stream_new.read(buffer_new, result_file_length);
+
+  START_PROFILE(allocTables);
+
+  // allocate hash/link tables
+  if (_hash_table == (PN_uint32 *)NULL) {
+    if (express_cat.is_debug()) {
+      express_cat.debug()
+        << "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n";
+    }
+    _hash_table = new PN_uint32[_HASHTABLESIZE];
+  }
 
   if (express_cat.is_debug()) {
     express_cat.debug()
@@ -944,7 +1036,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
   START_PROFILE(buildTables);
 
   // build hash and link tables for original file
-  build_hash_link_tables(buffer_orig, source_file_length, hash_table, link_table);
+  build_hash_link_tables(buffer_orig, source_file_length, _hash_table, link_table);
 
   END_PROFILE(buildTables, "building hash and link tables");
 
@@ -952,7 +1044,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
   START_PROFILE(buildPatchfile);
 
   PN_uint32 new_pos = 0;
-  PN_uint32 ADD_pos = new_pos; // this is the position for the start of ADD operations
+  PN_uint32 start_pos = new_pos; // this is the position for the start of ADD operations
 
   if(((PN_uint32) result_file_length) >= _footprint_length)
   {
@@ -962,7 +1054,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
       PN_uint32 COPY_pos;
       PN_uint16 COPY_length;
 
-      find_longest_match(new_pos, COPY_pos, COPY_length, hash_table, link_table,
+      find_longest_match(new_pos, COPY_pos, COPY_length, _hash_table, link_table,
         buffer_orig, source_file_length, buffer_new, result_file_length);
 
       // if no match or match not longer than footprint length, skip to next byte
@@ -971,22 +1063,16 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
         new_pos++;
       } else {
         // emit ADD for all skipped bytes
-        int num_skipped = (int)new_pos - (int)ADD_pos;
+        int num_skipped = (int)new_pos - (int)start_pos;
         if (express_cat.is_spam()) {
           express_cat.spam()
             << "build: num_skipped = " << num_skipped 
             << endl;
         }
-        last_copy_pos =
-          emit_add_and_copy(write_stream, num_skipped, &buffer_new[ADD_pos],
-                            COPY_length, COPY_pos + copy_offset,
-                            last_copy_pos, ADD_pos);
-        ADD_pos += num_skipped;
-        nassertr(ADD_pos == new_pos, last_copy_pos);
-
-        // skip past match in new_file
+        cache_add_and_copy(write_stream, num_skipped, &buffer_new[start_pos],
+                           COPY_length, COPY_pos + copy_offset);
         new_pos += (PN_uint32)COPY_length;
-        ADD_pos = new_pos;
+        start_pos = new_pos;
       }
     }
   }
@@ -994,28 +1080,28 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
   if (express_cat.is_spam()) {
     express_cat.spam()
       << "build: result_file_length = " << result_file_length
-      << " ADD_pos = " << ADD_pos
+      << " start_pos = " << start_pos
       << endl;
   }
 
   // are there still more bytes left in the new file?
-  if (ADD_pos != result_file_length) {
+  if (start_pos != result_file_length) {
     // emit ADD for all remaining bytes
 
-    PN_uint32 remaining_bytes = result_file_length - ADD_pos;
-    last_copy_pos =
-      emit_add_and_copy(write_stream, remaining_bytes, &buffer_new[ADD_pos], 
-                        0, last_copy_pos, last_copy_pos, ADD_pos);
-    ADD_pos += remaining_bytes;
-    nassertr(ADD_pos == result_file_length, last_copy_pos);
+    PN_uint32 remaining_bytes = result_file_length - start_pos;
+    cache_add_and_copy(write_stream, remaining_bytes, &buffer_new[start_pos], 
+                       0, 0);
+    start_pos += remaining_bytes;
   }
 
   END_PROFILE(buildPatchfile, "building patch file");
 
-  delete[] hash_table;
   delete[] link_table;
 
-  return last_copy_pos;
+  delete[] buffer_orig;
+  delete[] buffer_new;
+
+  return true;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -1028,36 +1114,33 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
 //               much faster for large Multifiles that contain many
 //               small subfiles.
 ////////////////////////////////////////////////////////////////////
-PN_uint32 Patchfile::
+bool Patchfile::
 compute_mf_patches(ostream &write_stream, 
-                   char *buffer_orig, PN_uint32 source_file_length,
-                   char *buffer_new, PN_uint32 result_file_length) {
-  string string_orig(buffer_orig, source_file_length);
-  string string_new(buffer_new, result_file_length);
-  istringstream strm_orig(string_orig);
-  istringstream strm_new(string_new);
-
+                   istream &stream_orig, istream &stream_new) {
   Multifile mf_orig, mf_new;
-  if (!mf_orig.open_read(&strm_orig) ||
-      !mf_new.open_read(&strm_new)) {
+  if (!mf_orig.open_read(&stream_orig) ||
+      !mf_new.open_read(&stream_new)) {
     express_cat.error()
       << "Input multifiles appear to be corrupt.\n";
-    return 0;
+    return false;
   }
 
   if (mf_new.needs_repack()) {
     express_cat.error()
       << "Input multifiles need to be repacked.\n";
-    return 0;
+    return false;
   }
 
   // First, compute the patch for the header / index.
 
-  PN_uint32 last_copy_pos =
-    compute_patches(write_stream, 0, 0,
-                    buffer_orig, (PN_uint32)mf_orig.get_index_end(),
-                    buffer_new, (PN_uint32)mf_new.get_index_end());
-  PN_uint32 add_pos = (PN_uint32)mf_new.get_index_end();
+  {
+    ISubStream index_orig(&stream_orig, 0, mf_orig.get_index_end());
+    ISubStream index_new(&stream_new, 0, mf_new.get_index_end());
+    if (!compute_patches(write_stream, 0, index_orig, index_new)) {
+      return false;
+    }
+    nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_index_end(), false);
+  }
 
   // Now walk through each subfile in the new multifile.  If a
   // particular subfile exists in both source files, we compute the
@@ -1066,54 +1149,128 @@ compute_mf_patches(ostream &write_stream,
   // never even notice this case).
   int new_num_subfiles = mf_new.get_num_subfiles();
   for (int ni = 0; ni < new_num_subfiles; ++ni) {
-    nassertr(add_pos == mf_new.get_subfile_internal_start(ni), last_copy_pos);
+    nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_subfile_internal_start(ni), false);
     string name = mf_new.get_subfile_name(ni);
     int oi = mf_orig.find_subfile(name);
+
+    /*
+    if (oi < 0) {
+      string new_ext = Filename(name).get_extension();
+      if (new_ext != "jpg") {
+        // This is a newly-added subfile.  Look for another subfile with
+        // the same extension, so we can generate a patch against that
+        // other subfile--the idea is that there are likely to be
+        // similar byte sequences in files with the same extension, so
+        // "patching" a new subfile against a different subfile may come
+        // out smaller than baldly adding the new subfile.
+        size_t new_size = mf_new.get_subfile_internal_length(ni);
+        
+        int orig_num_subfiles = mf_orig.get_num_subfiles();
+        size_t best_size = 0;
+        for (int i = 0; i < orig_num_subfiles; ++i) {
+          string orig_ext = Filename(mf_orig.get_subfile_name(i)).get_extension();
+          if (orig_ext == new_ext) {
+            size_t orig_size = mf_orig.get_subfile_internal_length(i);
+            
+            // Find the smallest candidate that is no smaller than our
+            // target file.  If all the candidates are smaller than our
+            // target file, choose the largest of them.
+            if ((best_size < new_size && orig_size > best_size) ||
+                (best_size >= new_size && orig_size >= new_size && orig_size < best_size)) {
+              best_size = orig_size;
+              oi = i;
+            }
+          }
+        }
+      }
+    }
+    */
+
     if (oi < 0) {
-      // This subfile exists in the new file, but not in the original
-      // file.  Trivially add it.
+      // This is a newly-added subfile, and we didn't find another
+      // subfile with a matching extension.  Add it the hard way.
       express_cat.info()
         << "Adding subfile " << mf_new.get_subfile_name(ni) << "\n";
-      PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni);
-      PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni);
-      last_copy_pos =
-        emit_add_and_copy(write_stream, new_size, &buffer_new[new_start],
-                          0, last_copy_pos, last_copy_pos, add_pos);
-      add_pos += new_size;
+
+      streampos new_start = mf_new.get_subfile_internal_start(ni);
+      size_t new_size = mf_new.get_subfile_internal_length(ni);
+      char *buffer_new = new char[new_size];
+      stream_new.seekg(new_start, ios::beg);
+      stream_new.read(buffer_new, new_size);
+      cache_add_and_copy(write_stream, new_size, buffer_new, 0, 0);
+      delete[] buffer_new;
 
     } else {
       // This subfile exists in both the original and the new files.
       // Patch it.
-      PN_uint32 orig_start = (PN_uint32)mf_orig.get_subfile_internal_start(oi);
-      PN_uint32 orig_size = (PN_uint32)mf_orig.get_subfile_internal_length(oi);
-      PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni);
-      PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni);
-      if (orig_size == new_size &&
-          memcmp(&buffer_orig[orig_start], &buffer_new[new_start], new_size) == 0) {
-        // Actually, the subfile is unchanged; just emit it.
+
+      streampos orig_start = mf_orig.get_subfile_internal_start(oi);
+      size_t orig_size = mf_orig.get_subfile_internal_length(oi);
+      ISubStream subfile_orig(&stream_orig, orig_start, orig_start + (streampos)orig_size);
+
+      streampos new_start = mf_new.get_subfile_internal_start(ni);
+      size_t new_size = mf_new.get_subfile_internal_length(ni);
+      ISubStream subfile_new(&stream_new, new_start, new_start + (streampos)new_size);
+
+      bool is_unchanged = false;
+      if (orig_size == new_size) {
+        HashVal hash_orig, hash_new;
+        hash_orig.hash_stream(subfile_orig);
+        hash_new.hash_stream(subfile_new);
+
+        if (hash_orig == hash_new) {
+          // Actually, the subfile is unchanged; just emit it.
+          is_unchanged = true;
+        }
+      }
+
+      if (is_unchanged) {
         if (express_cat.is_debug()) {
           express_cat.debug()
-            << "Keeping subfile " << mf_new.get_subfile_name(ni) << "\n";
+            << "Keeping subfile " << mf_new.get_subfile_name(ni);
+          if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
+            express_cat.debug(false)
+              << " (identical to " << mf_orig.get_subfile_name(oi) << ")";
+          }
+          express_cat.debug(false) << "\n";
         }
-        last_copy_pos =
-          emit_add_and_copy(write_stream, 0, NULL, 
-                            new_size, orig_start, last_copy_pos,
-                            add_pos);
-                  
+        cache_add_and_copy(write_stream, 0, NULL, 
+                           orig_size, orig_start);
+
       } else {
         express_cat.info()
-          << "Patching subfile " << mf_new.get_subfile_name(ni) << "\n";
-        last_copy_pos = 
-          compute_patches(write_stream, last_copy_pos, orig_start,
-                          &buffer_orig[orig_start], orig_size,
-                          &buffer_new[new_start], new_size);
+          << "Patching subfile " << mf_new.get_subfile_name(ni);
+        if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
+          express_cat.info(false)
+            << " (against " << mf_orig.get_subfile_name(oi) << ")";
+        }
+        express_cat.info(false) << "\n";
+
+        if (!compute_patches(write_stream, orig_start,
+                             subfile_orig, subfile_new)) {
+          return false;
+        }
+
+        /*
+        // Simply copy the new file; don't attempt to patch.
+        if (express_cat.is_debug()) {
+          express_cat.debug()
+            << "Copying subfile " << mf_new.get_subfile_name(ni) << "\n";
+        }
+        streampos new_start = mf_new.get_subfile_internal_start(ni);
+        size_t new_size = mf_new.get_subfile_internal_length(ni);
+        char *buffer_new = new char[new_size];
+        stream_new.seekg(new_start, ios::beg);
+        stream_new.read(buffer_new, new_size);
+        cache_add_and_copy(write_stream, new_size, buffer_new,
+                           0, 0);
+        delete[] buffer_new;
+        */
       }
-      add_pos += new_size;
     }
   }
 
-  nassertr(add_pos == result_file_length, last_copy_pos);
-  return last_copy_pos;
+  return true;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -1134,8 +1291,6 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
 
   START_PROFILE(overall);
 
-  START_PROFILE(readFiles);
-
   // Open the original file for read
   ifstream stream_orig;
   file_orig.set_binary();
@@ -1162,73 +1317,67 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
     return false;
   }
 
-  // read in original file
-  stream_orig.seekg(0, ios::end);
-  PN_uint32 source_file_length = stream_orig.tellg();
-  if (express_cat.is_debug()) {
-    express_cat.debug()
-      << "Allocating " << source_file_length << " bytes to read " 
-      << file_orig << "\n";
-  }
-
-  char *buffer_orig = new char[source_file_length];
-  stream_orig.seekg(0, ios::beg);
-  stream_orig.read(buffer_orig, source_file_length);
-
-  // read in new file
-  stream_new.seekg(0, ios::end);
-  PN_uint32 result_file_length = stream_new.tellg();
-  if (express_cat.is_debug()) {
-    express_cat.debug()
-      << "Allocating " << result_file_length << " bytes to write " 
-      << file_new << "\n";
-  }
-
-  char *buffer_new = new char[result_file_length];
-  stream_new.seekg(0, ios::beg);
-  stream_new.read(buffer_new, result_file_length);
-
-  // close the original and new files (we have em in memory)
-  stream_orig.close();
-  stream_new.close();
+  _last_copy_pos = 0;
+  _add_pos = 0;
+  _cache_add_data = string();
+  _cache_copy_start = 0;
+  _cache_copy_length = 0;
 
-  END_PROFILE(readFiles, "reading files");
-
-  write_header(write_stream, buffer_orig, source_file_length,
-               buffer_new, result_file_length);
-
-  PN_uint32 last_copy_pos;
+  write_header(write_stream, stream_orig, stream_new);
 
   // Check whether our input files are Panda multifiles.
   bool is_multifile = false;
   if (_allow_multifile) {
     if (file_orig.get_extension() == "mf" || file_new.get_extension() == "mf") {
+      // Read the first n bytes of both files for the Multifile magic
+      // number.
       string magic_number = Multifile::get_magic_number();
-      if (source_file_length > magic_number.size() &&
-          result_file_length > magic_number.size() &&
-          memcmp(buffer_orig, magic_number.data(), magic_number.size()) == 0 &&
-          memcmp(buffer_new, magic_number.data(), magic_number.size()) == 0) {
-        is_multifile = true;
+      char *buffer = new char[magic_number.size()];
+      stream_orig.seekg(0, ios::beg);
+      stream_orig.read(buffer, magic_number.size());
+      
+      if (stream_orig.gcount() == magic_number.size() &&
+          memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
+        stream_new.seekg(0, ios::beg);
+        stream_new.read(buffer, magic_number.size());
+        if (stream_new.gcount() == magic_number.size() &&
+            memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
+          is_multifile = true;
+        }
       }
+      delete[] buffer;
     }
   }
 
   if (is_multifile) {
-    last_copy_pos =
-      compute_mf_patches(write_stream, buffer_orig, source_file_length,
-                         buffer_new, result_file_length);
+    if (express_cat.is_debug()) {
+      express_cat.debug()
+        << "Input files appear to be Panda Multifiles.\n";
+    }
+    if (!compute_mf_patches(write_stream, stream_orig, stream_new)) {
+      return false;
+    }
   } else {
-    last_copy_pos =
-      compute_patches(write_stream, 0, 0,
-                      buffer_orig, source_file_length,
-                      buffer_new, result_file_length);
+    if (express_cat.is_debug()) {
+      express_cat.debug()
+        << "Input files are NOT Panda Multifiles.\n";
+    }
+    if (!compute_patches(write_stream, 0, stream_orig, stream_new)) {
+      return false;
+    }
   }
 
-  write_terminator(write_stream, result_file_length, last_copy_pos);
+  write_terminator(write_stream);
 
   END_PROFILE(overall, "total patch building operation");
 
-  return (last_copy_pos != 0);
+  if (express_cat.is_debug()) {
+    express_cat.debug()
+      << "Patch file will generate " << _add_pos << "-byte file.\n";
+  }
+  //  nassertr(_add_pos == result_file_length, false);
+
+  return (_last_copy_pos != 0);
 }
 
 #endif // HAVE_OPENSSL

panda/src/express/patchfile.h

@@ -94,30 +94,24 @@ private:
   PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length,
     PN_uint32 min_length);
 
-  void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
-                PN_uint32 ADD_pos);
-  PN_uint32 emit_COPY(ostream &write_stream, PN_uint32 length, 
-                      PN_uint32 COPY_pos, PN_uint32 last_copy_pos,
-                      PN_uint32 ADD_pos);
-  PN_uint32 emit_add_and_copy(ostream &write_stream, 
-                              PN_uint32 add_length, const char *add_buffer,
-                              PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos,
-                              PN_uint32 add_pos);
-
+  void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer);
+  void emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos);
+  void emit_add_and_copy(ostream &write_stream, 
+                         PN_uint32 add_length, const char *add_buffer,
+                         PN_uint32 copy_length, PN_uint32 copy_pos);
+  void cache_add_and_copy(ostream &write_stream, 
+                          PN_uint32 add_length, const char *add_buffer,
+                          PN_uint32 copy_length, PN_uint32 copy_pos);
+  void cache_flush(ostream &write_stream);
 
   void write_header(ostream &write_stream, 
-                    char *buffer_orig, PN_uint32 source_file_length,
-                    char *buffer_new, PN_uint32 result_file_length);
-  void write_terminator(ostream &write_stream, PN_uint32 result_file_length,
-                        PN_uint32 last_copy_pos);
-
-  PN_uint32 compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
-                            PN_uint32 copy_offset,
-                            char *buffer_orig, PN_uint32 source_file_length,
-                            char *buffer_new, PN_uint32 result_file_length);
-  PN_uint32 compute_mf_patches(ostream &write_stream, 
-                               char *buffer_orig, PN_uint32 source_file_length,
-                               char *buffer_new, PN_uint32 result_file_length);
+                    istream &stream_orig, istream &stream_new);
+  void write_terminator(ostream &write_stream);
+
+  bool compute_patches(ostream &write_stream, PN_uint32 copy_offset,
+                       istream &stream_orig, istream &stream_new);
+  bool compute_mf_patches(ostream &write_stream, 
+                          istream &stream_orig, istream &stream_new);
 
   static const PN_uint32 _HASH_BITS;
   static const PN_uint32 _HASHTABLESIZE;
@@ -129,6 +123,15 @@ private:
   bool _allow_multifile;
   PN_uint32 _footprint_length;
 
+  PN_uint32 *_hash_table;
+
+  PN_uint32 _add_pos;
+  PN_uint32 _last_copy_pos;
+
+  string _cache_add_data;
+  PN_uint32 _cache_copy_start;
+  PN_uint32 _cache_copy_length;
+
 protected:
   PT(Buffer) _buffer; // this is the work buffer for apply -- used to prevent virtual memory swapping
 

panda/src/framework/windowFramework.cxx

@@ -190,7 +190,8 @@ open_window(const WindowProperties &props, GraphicsEngine *engine,
 
     if (show_frame_rate_meter) {
       _frame_rate_meter = new FrameRateMeter("frame_rate_meter");
-      _frame_rate_meter->setup_window(_window);
+      //      _frame_rate_meter->setup_window(_window);
+      _panda_framework->get_models().attach_new_node(_frame_rate_meter);
     }
   }