support compression in multifiles

panda/src/downloadertools/multify.cxx

@@ -24,6 +24,7 @@
 #endif
 #include "multifile.h"
 #include "filename.h"
+#include "pset.h"
 #include <stdio.h>
 
 
@@ -32,12 +33,15 @@ bool append = false;      // -r
 bool list = false;        // -t
 bool extract = false;     // -x
 bool verbose = false;     // -v
+bool compress = false;    // -z
+int default_compression_level = 6;
 Filename multifile_name;  // -f
 bool got_multifile_name = false;
 bool to_stdout = false;   // -O
 Filename chdir_to;        // -C
 bool got_chdir_to = false;
 size_t scale_factor = 0;  // -F
+pset<string> dont_compress; // -Z
 
 void 
 usage() {
@@ -73,6 +77,25 @@ is_named(const string &subfile_name, int argc, char *argv[]) {
   return false;
 }
 
+int
+get_compression_level(const Filename &subfile_name) {
+  // Returns the appropriate compression level for the named file.
+  if (!compress) {
+    // Don't compress anything.
+    return 0;
+  }
+
+  string ext = subfile_name.get_extension();
+  if (dont_compress.find(ext) != dont_compress.end()) {
+    // This extension is listed on the -Z parameter list; don't
+    // compress it.
+    return 0;
+  }
+
+  // Go ahead and compress this file.
+  return default_compression_level;
+}
+
 bool
 add_directory(Multifile &multifile, const Filename &directory_name) {
   vector_string files;
@@ -95,7 +118,8 @@ add_directory(Multifile &multifile, const Filename &directory_name) {
 
     } else {
       string new_subfile_name =
-        multifile.add_subfile(subfile_name, subfile_name);
+        multifile.add_subfile(subfile_name, subfile_name,
+                              get_compression_level(subfile_name));
       if (new_subfile_name.empty()) {
         cerr << "Unable to add " << subfile_name << ".\n";
         okflag = false;
@@ -144,7 +168,8 @@ add_files(int argc, char *argv[]) {
 
     } else {
       string new_subfile_name =
-        multifile.add_subfile(subfile_name, subfile_name);
+        multifile.add_subfile(subfile_name, subfile_name,
+                              get_compression_level(subfile_name));
       if (new_subfile_name.empty()) {
         cerr << "Unable to add " << subfile_name << ".\n";
         okflag = false;
@@ -228,9 +253,19 @@ list_files(int argc, char *argv[]) {
     for (int i = 0; i < num_subfiles; i++) {
       string subfile_name = multifile.get_subfile_name(i);
       if (is_named(subfile_name, argc, argv)) {
-        printf("%12d  %s\n", 
-               multifile.get_subfile_length(i),
-               subfile_name.c_str());
+        if (multifile.is_subfile_compressed(i)) {
+          double ratio = 
+            (double)multifile.get_subfile_compressed_length(i) /
+            (double)multifile.get_subfile_length(i);
+          printf("%12d %3.0f%%  %s\n",
+                 multifile.get_subfile_length(i),
+                 100.0 - ratio * 100.0,
+                 subfile_name.c_str());
+        } else {
+          printf("%12d       %s\n", 
+                 multifile.get_subfile_length(i),
+                 subfile_name.c_str());
+        }
       }
     }
     fflush(stdout);
@@ -252,6 +287,21 @@ list_files(int argc, char *argv[]) {
   return true;
 }
 
+void
+tokenize_extensions(const string &str, pset<string> &extensions) {
+  size_t p = 0;
+  while (p < str.length()) {
+    size_t q = str.find_first_of(",", p);
+    if (q == string::npos) {
+      extensions.insert(str.substr(p));
+      return;
+    }
+    extensions.insert(str.substr(p, q - p));
+    p = q + 1;
+  }
+  extensions.insert(string());
+}
+
 int
 main(int argc, char *argv[]) {
   if (argc < 2) {
@@ -270,9 +320,12 @@ main(int argc, char *argv[]) {
     }
   }
 
+  // Default extensions not to compress.  May be overridden with -Z.
+  string dont_compress_str = "jpg,mp3";
+
   extern char *optarg;
   extern int optind;
-  static const char *optflags = "crtxvf:OC:F:h";
+  static const char *optflags = "crtxvz123456789Z:f:OC:F:h";
   int flag = getopt(argc, argv, optflags);
   Filename rel_path;
   while (flag != EOF) {
@@ -292,6 +345,48 @@ main(int argc, char *argv[]) {
     case 'v':
       verbose = true;
       break;
+    case 'z':
+      compress = true;
+      break;
+    case '1':
+      default_compression_level = 1;
+      compress = true;
+      break;
+    case '2':
+      default_compression_level = 2;
+      compress = true;
+      break;
+    case '3':
+      default_compression_level = 3;
+      compress = true;
+      break;
+    case '4':
+      default_compression_level = 4;
+      compress = true;
+      break;
+    case '5':
+      default_compression_level = 5;
+      compress = true;
+      break;
+    case '6':
+      default_compression_level = 6;
+      compress = true;
+      break;
+    case '7':
+      default_compression_level = 7;
+      compress = true;
+      break;
+    case '8':
+      default_compression_level = 8;
+      compress = true;
+      break;
+    case '9':
+      default_compression_level = 9;
+      compress = true;
+      break;
+    case 'Z':
+      dont_compress_str = optarg;
+      break;
     case 'f':
       multifile_name = Filename::from_os_specific(optarg);
       got_multifile_name = true;
@@ -343,21 +438,14 @@ main(int argc, char *argv[]) {
   }
 
   if (!got_multifile_name) {
-    if (argc <= 1) {
-      usage();
-      return 1;
-    }
-
-    // For now, we allow -f to be omitted, and use the first argument
-    // as the archive name, for backward compatibility.  Later we will
-    // remove this.
-    multifile_name = Filename::from_os_specific(argv[1]);
-    cerr << "Warning: using " << multifile_name
-         << " as archive name.  Use -f in the future to specify this.\n";
-    argc--;
-    argv++;
+    cerr << "Multifile name not specified.\n";
+    usage();
+    return 1;
   }
 
+  // Split out the extensions named by -Z into different words.
+  tokenize_extensions(dont_compress_str, dont_compress);
+
   bool okflag = true;
   if (create || append) {
     okflag = add_files(argc, argv);

panda/src/express/Sources.pp

@@ -4,7 +4,7 @@
 
 #begin lib_target
   #define TARGET express
-  #define USE_PACKAGES nspr crypto net
+  #define USE_PACKAGES nspr crypto net zlib
   
   #define COMBINED_SOURCES $[TARGET]_composite1.cxx $[TARGET]_composite2.cxx
 
@@ -44,6 +44,7 @@
     typeRegistry.I typeRegistry.h \
     typeRegistryNode.I typeRegistryNode.h \
     vector_uchar.h \
+    zStream.I zStream.h zStreamBuf.h \
     $[if $[HAVE_CRYPTO], \
        crypto_utils.cxx crypto_utils.h patchfile.I \
        patchfile.cxx patchfile.h ]
@@ -66,7 +67,8 @@
     subStream.cxx subStreamBuf.cxx \
     trueClock.cxx typeHandle.cxx \
     typedObject.cxx typedReferenceCount.cxx \
-    typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx
+    typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx \
+    zStream.cxx zStreamBuf.cxx
 
   #define INSTALL_HEADERS  \
     bigEndian.h buffer.I buffer.h checksumHashGenerator.I  \
@@ -98,7 +100,8 @@
     typedReferenceCount.h typedef.h \
     typeRegistry.I typeRegistry.h \
     typeRegistryNode.I typeRegistryNode.h \
-    vector_uchar.h
+    vector_uchar.h \
+    zStream.I zStream.h zStreamBuf.h
 
   #define IGATESCAN all
 
@@ -125,3 +128,17 @@
   #define OTHER_LIBS $[OTHER_LIBS] pystub
 
 #end test_bin_target
+
+
+#if $[HAVE_ZLIB]
+#begin test_bin_target
+  #define TARGET test_zstream
+  #define USE_PACKAGES zlib
+  #define LOCAL_LIBS $[LOCAL_LIBS] express
+  #define OTHER_LIBS pystub
+
+  #define SOURCES \
+    test_zstream.cxx
+
+#end test_bin_target
+#endif

panda/src/express/express_composite2.cxx

@@ -19,4 +19,6 @@
 #include "typeRegistry.cxx"
 #include "typeRegistryNode.cxx"
 #include "vector_uchar.cxx"
+#include "zStream.cxx"
+#include "zStreamBuf.cxx"
 

panda/src/express/multifile.I

@@ -120,6 +120,7 @@ Subfile() {
   _data_length = 0;
   _source = (istream *)NULL;
   _flags = 0;
+  _compression_level = 0;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/express/multifile.cxx

@@ -21,6 +21,7 @@
 #include "config_express.h"
 #include "streamWriter.h"
 #include "streamReader.h"
+#include "zStream.h"
 
 #include <algorithm>
 
@@ -63,6 +64,9 @@ const int Multifile::_current_minor_ver = 0;
 //   uint32     The address of this subfile's data record.
 //   uint32     The length in bytes of this subfile's data record.
 //   uint16     The Subfile::_flags member.
+//  [uint32]    The original, uncompressed length of the subfile, if it
+//               is compressed.  This field is only present if the
+//               SF_compressed bit is set in _flags.
 //   uint16     The length in bytes of the subfile's name.
 //   char[n]    The subfile's name.
 //
@@ -295,7 +299,8 @@ set_scale_factor(size_t scale_factor) {
 //               been modified slightly), or empty string on failure.
 ////////////////////////////////////////////////////////////////////
 string Multifile::
-add_subfile(const string &subfile_name, const Filename &filename) {
+add_subfile(const string &subfile_name, const Filename &filename,
+            int compression_level) {
   nassertr(is_write_valid(), string());
 
   if (!filename.exists()) {
@@ -305,7 +310,7 @@ add_subfile(const string &subfile_name, const Filename &filename) {
   subfile->_source_filename = filename;
   subfile->_source_filename.set_binary();
 
-  return add_new_subfile(subfile_name, subfile);
+  return add_new_subfile(subfile_name, subfile, compression_level);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -670,12 +675,41 @@ get_subfile_name(int index) const {
 ////////////////////////////////////////////////////////////////////
 //     Function: Multifile::get_subfile_length
 //       Access: Published
-//  Description: Returns the data length of the nth subfile.  This
-//               might return 0 if the subfile has recently been added
-//               and flush() has not yet been called.
+//  Description: Returns the uncompressed data length of the nth
+//               subfile.  This might return 0 if the subfile has
+//               recently been added and flush() has not yet been
+//               called.
 ////////////////////////////////////////////////////////////////////
 size_t Multifile::
 get_subfile_length(int index) const {
+  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
+  return _subfiles[index]->_uncompressed_length;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::is_subfile_compressed
+//       Access: Published
+//  Description: Returns true if the indicated subfile has been
+//               compressed when stored within the archive, false
+//               otherwise.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+is_subfile_compressed(int index) const {
+  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
+  return (_subfiles[index]->_flags & SF_compressed) != 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::get_subfile_compressed_length
+//       Access: Published
+//  Description: Returns the number of bytes the indicated subfile
+//               consumes within the archive.  For compressed
+//               subfiles, this will generally be smaller than
+//               get_subfile_length(); for noncompressed subfiles, it
+//               will be equal.
+////////////////////////////////////////////////////////////////////
+size_t Multifile::
+get_subfile_compressed_length(int index) const {
   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
   return _subfiles[index]->_data_length;
 }
@@ -818,7 +852,7 @@ open_read_write(iostream *multifile_stream) {
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Multifile::add_subfile
-//       Access: Published
+//       Access: Public
 //  Description: Adds a file on disk as a subfile to the Multifile.
 //               The indicated istream will be read and its contents
 //               added to the Multifile at the next call to flush().
@@ -827,13 +861,14 @@ open_read_write(iostream *multifile_stream) {
 //               been modified slightly), or empty string on failure.
 ////////////////////////////////////////////////////////////////////
 string Multifile::
-add_subfile(const string &subfile_name, istream *subfile_data) {
+add_subfile(const string &subfile_name, istream *subfile_data,
+            int compression_level) {
   nassertr(is_write_valid(), string());
 
   Subfile *subfile = new Subfile;
   subfile->_source = subfile_data;
 
-  return add_new_subfile(subfile_name, subfile);
+  return add_new_subfile(subfile_name, subfile, compression_level);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -903,9 +938,17 @@ open_read_subfile(int index) {
   // Return an ISubStream object that references into the open
   // Multifile istream.
   nassertr(subfile->_data_start != (streampos)0, new fstream);
-  ISubStream *stream = new ISubStream;
-  stream->open(_read, subfile->_data_start,
-               subfile->_data_start + (streampos)subfile->_data_length); 
+  istream *stream = 
+    new ISubStream(_read, subfile->_data_start,
+                   subfile->_data_start + (streampos)subfile->_data_length); 
+  
+  if ((subfile->_flags & SF_compressed) != 0) {
+    // Oops, the subfile is compressed.  So actually, return an
+    // IDecompressStream that wraps around the ISubStream.
+    IDecompressStream *wrapper = new IDecompressStream(stream, true);
+    stream = wrapper;
+  }
+
   return stream;
 }
 
@@ -937,7 +980,13 @@ pad_to_streampos(streampos fpos) {
 //               Multifile.
 ////////////////////////////////////////////////////////////////////
 string Multifile::
-add_new_subfile(const string &subfile_name, Subfile *subfile) {
+add_new_subfile(const string &subfile_name, Subfile *subfile,
+                int compression_level) {
+  if (compression_level != 0) {
+    subfile->_flags |= SF_compressed;
+    subfile->_compression_level = compression_level;
+  }
+
   if (_next_index != (streampos)0) {
     // If we're adding a Subfile to an already-existing Multifile, we
     // will eventually need to repack the file.
@@ -1160,6 +1209,11 @@ read_index(istream &read, streampos fpos, Multifile *multifile) {
   _data_start = multifile->word_to_streampos(reader.get_uint32());
   _data_length = reader.get_uint32();
   _flags = reader.get_uint16();
+  if ((_flags & SF_compressed) != 0) {
+    _uncompressed_length = reader.get_uint32();
+  } else {
+    _uncompressed_length = _data_length;
+  }
   size_t name_length = reader.get_uint16();
   if (read.eof() || read.fail()) {
     _flags |= SF_index_invalid;
@@ -1207,6 +1261,9 @@ write_index(ostream &write, streampos fpos, Multifile *multifile) {
   dg.add_uint32(multifile->streampos_to_word(_data_start));
   dg.add_uint32(_data_length);
   dg.add_uint16(_flags);
+  if ((_flags & SF_compressed) != 0) {
+    dg.add_uint32(_uncompressed_length);
+  }
   dg.add_uint16(_name.length());
 
   // For no real good reason, we'll invert all the bits in the name.
@@ -1243,8 +1300,9 @@ write_index(ostream &write, streampos fpos, Multifile *multifile) {
 //               effective end of the file.  Returns the position
 //               within the file of the next data record.
 //
-//               The _data_start and _data_length members are updated
-//               by this operation.
+//               The _data_start, _data_length, and
+//               _uncompressed_length members are updated by this
+//               operation.
 //
 //               If the "read" pointer is non-NULL, it is the readable
 //               istream of a Multifile in which the Subfile might
@@ -1265,6 +1323,7 @@ write_data(ostream &write, istream *read, streampos fpos) {
         << "Unable to read " << _source_filename << ".\n";
       _flags |= SF_data_invalid;
       _data_length = 0;
+      _uncompressed_length = 0;
     } else {
       source = &source_file;
     }
@@ -1295,12 +1354,30 @@ write_data(ostream &write, istream *read, streampos fpos) {
   } else {
     // We do have source data.  Copy it in, and also measure its
     // length.
-    _data_length = 0;
-    int byte = source->get();
-    while (!source->eof() && !source->fail()) {
-      _data_length++;
-      write.put(byte);
-      byte = source->get();
+    if ((_flags & SF_compressed) != 0) {
+      // Write it compressed.
+      streampos write_start = write.tellp();
+      _uncompressed_length = 0;
+      OCompressStream zstream(&write, false, _compression_level);
+      int byte = source->get();
+      while (!source->eof() && !source->fail()) {
+        _uncompressed_length++;
+        zstream.put(byte);
+        byte = source->get();
+      }
+      zstream.close();
+      streampos write_end = write.tellp();
+      _data_length = (size_t)(write_end - write_start);
+    } else {
+      // Write it uncompressed.
+      _uncompressed_length = 0;
+      int byte = source->get();
+      while (!source->eof() && !source->fail()) {
+        _uncompressed_length++;
+        write.put(byte);
+        byte = source->get();
+      }
+      _data_length = _uncompressed_length;
     }
   }
 
@@ -1334,6 +1411,10 @@ rewrite_index_data_start(ostream &write, Multifile *multifile) {
   StreamWriter writer(write);
   writer.add_uint32(multifile->streampos_to_word(_data_start));
   writer.add_uint32(_data_length);
+  writer.add_uint16(_flags);
+  if ((_flags & SF_compressed) != 0) {
+    writer.add_uint32(_uncompressed_length);
+  }
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/express/multifile.h

@@ -57,7 +57,8 @@ PUBLISHED:
   void set_scale_factor(size_t scale_factor);
   INLINE size_t get_scale_factor() const;
 
-  string add_subfile(const string &subfile_name, const Filename &filename);
+  string add_subfile(const string &subfile_name, const Filename &filename,
+                     int compression_level);
   bool flush();
   bool repack();
 
@@ -69,6 +70,8 @@ PUBLISHED:
   void remove_subfile(int index);
   const string &get_subfile_name(int index) const;
   size_t get_subfile_length(int index) const;
+  bool is_subfile_compressed(int index) const;
+  size_t get_subfile_compressed_length(int index) const;
 
   void read_subfile(int index, Datagram &datagram);
   bool extract_subfile(int index, const Filename &filename);
@@ -81,7 +84,8 @@ public:
   bool open_read(istream *multifile_stream);
   bool open_write(ostream *multifile_stream);
   bool open_read_write(iostream *multifile_stream);
-  string add_subfile(const string &subfile_name, istream *subfile_data);
+  string add_subfile(const string &subfile_name, istream *subfile_data,
+                     int compression_level);
 
   bool extract_subfile_to(int index, ostream &out);
   istream *open_read_subfile(int index);
@@ -91,6 +95,7 @@ private:
     SF_deleted        = 0x0001,
     SF_index_invalid  = 0x0002,
     SF_data_invalid   = 0x0004,
+    SF_compressed     = 0x0008,
   };
 
   class Subfile {
@@ -112,9 +117,11 @@ private:
     streampos _index_start;
     streampos _data_start;
     size_t _data_length;
+    size_t _uncompressed_length;
     istream *_source;
     Filename _source_filename;
     int _flags;
+    int _compression_level;  // Not preserved on disk.
   };
 
   INLINE streampos word_to_streampos(size_t word) const;
@@ -122,7 +129,8 @@ private:
   INLINE streampos normalize_streampos(streampos fpos) const;
   streampos pad_to_streampos(streampos fpos);
 
-  string add_new_subfile(const string &subfile_name, Subfile *subfile);
+  string add_new_subfile(const string &subfile_name, Subfile *subfile,
+                         int compression_level);
   void clear_subfiles();
   bool read_index();
   bool write_header();

panda/src/express/subStream.I

@@ -26,6 +26,16 @@ INLINE ISubStream::
 ISubStream() : istream(&_buf) {
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: ISubStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE ISubStream::
+ISubStream(istream *source, streampos start, streampos end) : istream(&_buf) {
+  open(source, start, end);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: ISubStream::open
 //       Access: Public
@@ -39,10 +49,11 @@ ISubStream() : istream(&_buf) {
 //               If end is zero, it indicates that the ISubStream will
 //               continue until the end of the source stream.
 ////////////////////////////////////////////////////////////////////
-INLINE void ISubStream::
+INLINE ISubStream &ISubStream::
 open(istream *source, streampos start, streampos end) {
   clear(0);
   _buf.open(source, start, end);
+  return *this;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -51,8 +62,9 @@ open(istream *source, streampos start, streampos end) {
 //  Description: Resets the SubStream to empty, but does not actually
 //               close the source istream.
 ////////////////////////////////////////////////////////////////////
-INLINE void ISubStream::
+INLINE ISubStream &ISubStream::
 close() {
   _buf.close();
+  return *this;
 }
 

panda/src/express/subStream.h

@@ -37,9 +37,10 @@
 class EXPCL_PANDAEXPRESS ISubStream : public istream {
 public:
   INLINE ISubStream();
+  INLINE ISubStream(istream *source, streampos start, streampos end);
 
-  INLINE void open(istream *source, streampos start, streampos end);
-  INLINE void close();
+  INLINE ISubStream &open(istream *source, streampos start, streampos end);
+  INLINE ISubStream &close();
 
 private:
   SubStreamBuf _buf;

panda/src/express/test_zstream.cxx

@@ -0,0 +1,70 @@
+// Filename: test_zstream.cxx
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "pandabase.h"
+#include "zStream.h"
+#include "filename.h"
+
+void
+decompress(istream &source) {
+  IDecompressStream zstream(&source, false);
+
+  int ch = zstream.get();
+  while (!zstream.eof() && !zstream.fail()) {
+    cout.put(ch);
+    ch = zstream.get();
+  }
+}
+
+void
+compress(istream &source) {
+  OCompressStream zstream(&cout, false);
+
+  int ch = source.get();
+  while (!source.eof() && !source.fail()) {
+    zstream.put(ch);
+    ch = source.get();
+  }
+}
+
+int
+main(int argc, char *argv[]) {
+  if (argc != 2) {
+    cerr << "test_zstream file\n"
+         << "compresses file to standard output, or decompresses it if the\n"
+         << "filename ends in .pz.\n";
+    return (1);
+  }
+
+  Filename source_filename = argv[1];
+  source_filename.set_binary();
+
+  ifstream source;
+
+  if (!source_filename.open_read(source)) {
+    cerr << "Unable to open source " << source_filename << ".\n";
+    return (1);
+  }
+  if (source_filename.get_extension() == "pz") {
+    decompress(source);
+  } else {
+    compress(source);
+  }
+
+  return (0);
+}

panda/src/express/zStream.I

@@ -0,0 +1,108 @@
+// Filename: zStream.I
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: IDecompressStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE IDecompressStream::
+IDecompressStream() : istream(&_buf) {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IDecompressStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE IDecompressStream::
+IDecompressStream(istream *source, bool owns_source) : istream(&_buf) {
+  open(source, owns_source);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IDecompressStream::open
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE IDecompressStream &IDecompressStream::
+open(istream *source, bool owns_source) {
+  clear(0);
+  _buf.open_read(source, owns_source);
+  return *this;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IDecompressStream::close
+//       Access: Public
+//  Description: Resets the ZStream to empty, but does not actually
+//               close the source istream unless owns_source was true.
+////////////////////////////////////////////////////////////////////
+INLINE IDecompressStream &IDecompressStream::
+close() {
+  _buf.close_read();
+  return *this;
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: OCompressStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE OCompressStream::
+OCompressStream() : ostream(&_buf) {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: OCompressStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE OCompressStream::
+OCompressStream(ostream *dest, bool owns_dest, int compression_level) :
+  ostream(&_buf) 
+{
+  open(dest, owns_dest, compression_level);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: OCompressStream::open
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE OCompressStream &OCompressStream::
+open(ostream *dest, bool owns_dest, int compression_level) {
+  clear(0);
+  _buf.open_write(dest, owns_dest, compression_level);
+  return *this;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: OCompressStream::close
+//       Access: Public
+//  Description: Resets the ZStream to empty, but does not actually
+//               close the dest ostream unless owns_dest was true.
+////////////////////////////////////////////////////////////////////
+INLINE OCompressStream &OCompressStream::
+close() {
+  _buf.close_write();
+  return *this;
+}
+

panda/src/express/zStream.cxx

@@ -0,0 +1,19 @@
+// Filename: zStream.cxx
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "zStream.h"

panda/src/express/zStream.h

@@ -0,0 +1,86 @@
+// Filename: zStream.h
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef ZSTREAM_H
+#define ZSTREAM_H
+
+#include "pandabase.h"
+
+// This module is not compiled if zlib is not available.
+#ifdef HAVE_ZLIB
+
+#include "zStreamBuf.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : IDecompressStream
+// Description : An input stream object that uses zlib to decompress
+//               (inflate) the input from another source stream
+//               on-the-fly.
+//
+//               Attach an IDecompressStream to an existing istream that
+//               provides compressed data, and read the corresponding
+//               uncompressed data from the IDecompressStream.
+//
+//               Seeking is not supported.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS IDecompressStream : public istream {
+public:
+  INLINE IDecompressStream();
+  INLINE IDecompressStream(istream *source, bool owns_source);
+
+  INLINE IDecompressStream &open(istream *source, bool owns_source);
+  INLINE IDecompressStream &close();
+
+private:
+  ZStreamBuf _buf;
+};
+
+////////////////////////////////////////////////////////////////////
+//       Class : OCompressStream
+// Description : An input stream object that uses zlib to compress
+//               (deflate) data to another destination stream
+//               on-the-fly.
+//
+//               Attach an OCompressStream to an existing ostream that will
+//               accept compressed data, and write your uncompressed
+//               source data to the OCompressStream.
+//
+//               Seeking is not supported.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS OCompressStream : public ostream {
+public:
+  INLINE OCompressStream();
+  INLINE OCompressStream(ostream *dest, bool owns_dest, 
+                           int compression_level = 6);
+
+  INLINE OCompressStream &open(ostream *dest, bool owns_dest, 
+                                 int compression_level = 6);
+  INLINE OCompressStream &close();
+
+private:
+  ZStreamBuf _buf;
+};
+
+#include "zStream.I"
+
+#endif  // HAVE_ZLIB
+
+
+#endif
+
+

panda/src/express/zStreamBuf.cxx

@@ -0,0 +1,333 @@
+// Filename: zStreamBuf.cxx
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "zStreamBuf.h"
+
+#ifdef HAVE_ZLIB
+
+#ifndef HAVE_STREAMSIZE
+// Some compilers (notably SGI) don't define this for us
+typedef int streamsize;
+#endif /* HAVE_STREAMSIZE */
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+ZStreamBuf::
+ZStreamBuf() {
+  _source = (istream *)NULL;
+  _owns_source = false;
+  _dest = (ostream *)NULL;
+  _owns_dest = false;
+
+#ifdef WIN32_VC
+  // In spite of the claims of the MSDN Library to the contrary,
+  // Windows doesn't seem to provide an allocate() function, so we'll
+  // do it by hand.
+  char *buf = new char[4096];
+  char *ebuf = buf + 4096;
+  setg(buf, ebuf, ebuf);
+  setp(buf, ebuf, ebuf);
+
+#else
+  allocate();
+  setg(base(), ebuf(), ebuf());
+  setp(base(), ebuf());
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::Destructor
+//       Access: Public, Virtual
+//  Description:
+////////////////////////////////////////////////////////////////////
+ZStreamBuf::
+~ZStreamBuf() {
+  close_read();
+  close_write();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::open_read
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void ZStreamBuf::
+open_read(istream *source, bool owns_source) {
+  _source = source;
+  _owns_source = owns_source;
+
+  _z_source.next_in = Z_NULL;
+  _z_source.avail_in = 0;
+  _z_source.zalloc = Z_NULL;
+  _z_source.zfree = Z_NULL;
+  _z_source.opaque = Z_NULL;
+
+  int result = inflateInit(&_z_source);
+  if (result < 0) {
+    express_cat.warning()
+      << "zlib error " << result << " = " << _z_source.msg << "\n";
+    close_read();
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::close_read
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void ZStreamBuf::
+close_read() {
+  if (_source != (istream *)NULL) {
+
+    int result = inflateEnd(&_z_source);
+    if (result < 0) {
+      express_cat.warning()
+        << "zlib error " << result << " = " << _z_source.msg << "\n";
+    }
+
+    if (_owns_source) {
+      delete _source;
+      _owns_source = false;
+    }
+    _source = (istream *)NULL;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::open_write
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void ZStreamBuf::
+open_write(ostream *dest, bool owns_dest, int compression_level) {
+  _dest = dest;
+  _owns_dest = owns_dest;
+
+  _z_dest.zalloc = Z_NULL;
+  _z_dest.zfree = Z_NULL;
+  _z_dest.opaque = Z_NULL;
+
+  int result = deflateInit(&_z_dest, compression_level);
+  if (result < 0) {
+    express_cat.warning()
+      << "zlib error " << result << " = " << _z_dest.msg << "\n";
+    close_write();
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::close_write
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void ZStreamBuf::
+close_write() {
+  if (_dest != (ostream *)NULL) {
+    size_t n = pptr() - pbase();
+    write_chars(pbase(), n, Z_FINISH);
+    pbump(-(int)n);
+
+    int result = deflateEnd(&_z_dest);
+    if (result < 0) {
+      express_cat.warning()
+        << "zlib error " << result << " = " << _z_dest.msg << "\n";
+    }
+
+    if (_owns_dest) {
+      delete _dest;
+      _owns_dest = false;
+    }
+    _dest = (ostream *)NULL;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::overflow
+//       Access: Protected, Virtual
+//  Description: Called by the system ostream implementation when its
+//               internal buffer is filled, plus one character.
+////////////////////////////////////////////////////////////////////
+int ZStreamBuf::
+overflow(int ch) {
+  size_t n = pptr() - pbase();
+  if (n != 0) {
+    write_chars(pbase(), n, 0);
+    pbump(-(int)n);
+  }
+
+  if (ch != EOF) {
+    // Write one more character.
+    char c = ch;
+    write_chars(&c, 1, 0);
+  }
+
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::sync
+//       Access: Protected, Virtual
+//  Description: Called by the system iostream implementation to
+//               implement a flush operation.
+////////////////////////////////////////////////////////////////////
+int ZStreamBuf::
+sync() {
+  if (_source != (istream *)NULL) {
+    size_t n = egptr() - gptr();
+    gbump(n);
+  }
+
+  if (_dest != (ostream *)NULL) {
+    size_t n = pptr() - pbase();
+    write_chars(pbase(), n, Z_SYNC_FLUSH);
+    pbump(-(int)n);
+  }
+
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::underflow
+//       Access: Protected, Virtual
+//  Description: Called by the system istream implementation when its
+//               internal buffer needs more characters.
+////////////////////////////////////////////////////////////////////
+int ZStreamBuf::
+underflow() {
+  // Sometimes underflow() is called even if the buffer is not empty.
+  if (gptr() >= egptr()) {
+    size_t buffer_size = egptr() - eback();
+    gbump(-(int)buffer_size);
+
+    size_t num_bytes = buffer_size;
+    size_t read_count = read_chars(gptr(), buffer_size);
+
+    if (read_count != num_bytes) {
+      // Oops, we didn't read what we thought we would.
+      if (read_count == 0) {
+        return EOF;
+      }
+
+      // Slide what we did read to the top of the buffer.
+      nassertr(read_count < num_bytes, EOF);
+      size_t delta = num_bytes - read_count;
+      memmove(gptr() + delta, gptr(), read_count);
+      gbump(delta);
+    }
+  }
+
+  return (unsigned char)*gptr();
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::read_chars
+//       Access: Private
+//  Description: Gets some characters from the source stream.
+////////////////////////////////////////////////////////////////////
+size_t ZStreamBuf::
+read_chars(char *start, size_t length) {
+  static const size_t decompress_buffer_size = 4096;
+  char decompress_buffer[decompress_buffer_size];
+
+  _z_source.next_out = (Bytef *)start;
+  _z_source.avail_out = length;
+
+  int flush = (_source->eof() || _source->fail()) ? Z_FINISH : 0;
+
+  while (_z_source.avail_out > 0) {
+    if (_z_source.avail_in == 0 && flush == 0) {
+      _source->read(decompress_buffer, decompress_buffer_size);
+      size_t read_count = _source->gcount();
+      if (read_count == 0 || _source->eof() || _source->fail()) {
+        // End of input; tell zlib to expect to stop.
+        flush = Z_FINISH;
+      }
+        
+      _z_source.next_in = (Bytef *)decompress_buffer;
+      _z_source.avail_in = read_count;
+    }
+    int result = inflate(&_z_source, flush);
+    size_t bytes_read = length - _z_source.avail_out;
+
+    if (result == Z_STREAM_END) {
+      // Here's the end of the file.
+      return bytes_read;
+    }
+    if (result < 0) {
+      express_cat.warning()
+        << "zlib error " << result << " = " << _z_source.msg << "\n";
+      return bytes_read;
+    }
+  }
+
+  return length;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: ZStreamBuf::write_chars
+//       Access: Private
+//  Description: Sends some characters to the dest stream.  The flush
+//               parameter is passed to deflate().
+////////////////////////////////////////////////////////////////////
+void ZStreamBuf::
+write_chars(const char *start, size_t length, int flush) {
+  static const size_t compress_buffer_size = 4096;
+  char compress_buffer[compress_buffer_size];
+
+  _z_dest.next_in = (Bytef *)(char *)start;
+  _z_dest.avail_in = length;
+
+  _z_dest.next_out = (Bytef *)compress_buffer;
+  _z_dest.avail_out = compress_buffer_size;
+
+  int result = deflate(&_z_dest, flush);
+  if (result < 0 && result != Z_BUF_ERROR) {
+    express_cat.warning()
+      << "zlib error " << result << " = " << _z_dest.msg << "\n";
+  }
+
+  while (_z_dest.avail_in != 0) {
+    if (_z_dest.avail_out != compress_buffer_size) {
+      _dest->write(compress_buffer, compress_buffer_size - _z_dest.avail_out);
+      _z_dest.next_out = (Bytef *)compress_buffer;
+      _z_dest.avail_out = compress_buffer_size;
+    }
+    result = deflate(&_z_dest, flush);
+    if (result < 0) {
+      express_cat.warning()
+        << "zlib error " << result << " = " << _z_dest.msg << "\n";
+    }
+  }
+
+  while (_z_dest.avail_out != compress_buffer_size) {
+    _dest->write(compress_buffer, compress_buffer_size - _z_dest.avail_out);
+    _z_dest.next_out = (Bytef *)compress_buffer;
+    _z_dest.avail_out = compress_buffer_size;
+    result = deflate(&_z_dest, flush);
+    if (result < 0 && result != Z_BUF_ERROR) {
+      express_cat.warning()
+        << "zlib error " << result << " = " << _z_dest.msg << "\n";
+    }
+  }
+}
+
+#endif  // HAVE_ZLIB

panda/src/express/zStreamBuf.h

@@ -0,0 +1,67 @@
+// Filename: zStreamBuf.h
+// Created by:  drose (05Aug02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef ZSTREAMBUF_H
+#define ZSTREAMBUF_H
+
+#include "pandabase.h"
+
+// This module is not compiled if zlib is not available.
+#ifdef HAVE_ZLIB
+
+#include <zlib.h>
+
+////////////////////////////////////////////////////////////////////
+//       Class : ZStreamBuf
+// Description : The streambuf object that implements
+//               IDecompressStream and OCompressStream.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS ZStreamBuf : public streambuf {
+public:
+  ZStreamBuf();
+  virtual ~ZStreamBuf();
+
+  void open_read(istream *source, bool owns_source);
+  void close_read();
+
+  void open_write(ostream *dest, bool owns_dest, int compression_level);
+  void close_write();
+
+protected:
+  virtual int overflow(int c);
+  virtual int sync(void);
+  virtual int underflow(void);
+
+private:
+  size_t read_chars(char *start, size_t length);
+  void write_chars(const char *start, size_t length, int flush);
+
+private:
+  istream *_source;
+  bool _owns_source;
+
+  ostream *_dest;
+  bool _owns_dest;
+
+  z_stream _z_source;
+  z_stream _z_dest;
+};
+
+#endif  // HAVE_ZLIB
+
+#endif