use multifiles instead of tarfiles

direct/src/plugin/Sources.pp

@@ -18,6 +18,7 @@
     p3dFileDownload.h p3dFileDownload.I \
     p3dInstance.h p3dInstance.I \
     p3dInstanceManager.h p3dInstanceManager.I \
+    p3dMultifileReader.h p3dMultifileReader.I \
     p3dPackage.h p3dPackage.I \
     p3dSession.h p3dSession.I
 
@@ -27,6 +28,7 @@
     p3dFileDownload.cxx \
     p3dInstance.cxx \
     p3dInstanceManager.cxx \
+    p3dMultifileReader.cxx \
     p3dPackage.cxx \
     p3dSession.cxx
 

direct/src/plugin/make_package.py

@@ -27,10 +27,7 @@ Options:
 import sys
 import getopt
 import os
-import tarfile
-import gzip
-import stat
-import md5
+import zlib
 
 import direct
 from pandac.PandaModules import *
@@ -42,25 +39,15 @@ class FileSpec:
     def __init__(self, filename, pathname):
         self.filename = filename
         self.pathname = pathname
-        self.size = 0
-        self.timestamp = 0
-        self.hash = None
 
-        s = os.stat(self.pathname)
-        self.size = s[stat.ST_SIZE]
-        self.timestamp = s[stat.ST_MTIME]
+        self.size = pathname.getFileSize()
+        self.timestamp = pathname.getTimestamp()
 
-        m = md5.new()
-        f = open(self.pathname, 'rb')
-        data = f.read(4096)
-        while data:
-            m.update(data)
-            data = f.read(4096)
-        f.close()
-
-        self.hash = m.hexdigest()
+        hv = HashVal()
+        hv.hashFile(pathname)
+        self.hash = hv.asHex()
 
-    def get_params(self):
+    def getParams(self):
         return 'filename="%s" size=%s timestamp=%s hash="%s"' % (
             self.filename, self.size, self.timestamp, self.hash)
 
@@ -85,9 +72,11 @@ class PackageMaker:
 
         self.cleanDir(self.stageDir)
 
-        uncompressedArchiveBasename = '%s.tar' % (self.packageFullname)
-        uncompressedArchivePathname = os.path.join(self.stageDir, uncompressedArchiveBasename)
-        self.archive = tarfile.open(uncompressedArchivePathname, 'w')
+        uncompressedArchiveBasename = '%s.mf' % (self.packageFullname)
+        uncompressedArchivePathname = Filename(self.stageDir, uncompressedArchiveBasename)
+        self.archive = Multifile()
+        if not self.archive.openWrite(uncompressedArchivePathname):
+            raise IOError, "Couldn't open %s for writing" % (uncompressedArchivePathname)
 
         self.components = []
 
@@ -96,34 +85,37 @@ class PackageMaker:
 
         uncompressedArchive = FileSpec(uncompressedArchiveBasename, uncompressedArchivePathname)
 
-        compressedArchiveBasename = '%s.tgz' % (self.packageFullname)
-        compressedArchivePathname = os.path.join(self.stageDir, compressedArchiveBasename)
+        compressedArchiveBasename = '%s.mf.pz' % (self.packageFullname)
+        compressedArchivePathname = Filename(self.stageDir, compressedArchiveBasename)
 
         print "\ncompressing"
-        f = open(uncompressedArchivePathname, 'rb')
-        gz = gzip.open(compressedArchivePathname, 'w', 9)
-        data = f.read(4096)
+
+        source = open(uncompressedArchivePathname.toOsSpecific(), 'rb')
+        target = open(compressedArchivePathname.toOsSpecific(), 'w')
+        z = zlib.compressobj(9)
+        data = source.read(4096)
         while data:
-            gz.write(data)
-            data = f.read(4096)
-        gz.close()
-        f.close()
+            target.write(z.compress(data))
+            data = source.read(4096)
+        target.write(z.flush())
+        target.close()
+        source.close()
 
         compressedArchive = FileSpec(compressedArchiveBasename, compressedArchivePathname)
 
-        os.unlink(uncompressedArchivePathname)
+        uncompressedArchivePathname.unlink()
 
         descFileBasename = '%s.xml' % (self.packageFullname)
-        descFilePathname = os.path.join(self.stageDir, descFileBasename)
+        descFilePathname = Filename(self.stageDir, descFileBasename)
 
-        f = open(descFilePathname, 'w')
+        f = open(descFilePathname.toOsSpecific(), 'w')
         print >> f, '<?xml version="1.0" ?>'
         print >> f, ''
         print >> f, '<package name="%s" version="%s">' % (self.packageName, self.packageVersion)
-        print >> f, '  <uncompressed_archive %s />' % (uncompressedArchive.get_params())
-        print >> f, '  <compressed_archive %s />' % (compressedArchive.get_params())
+        print >> f, '  <uncompressed_archive %s />' % (uncompressedArchive.getParams())
+        print >> f, '  <compressed_archive %s />' % (compressedArchive.getParams())
         for file in self.components:
-            print >> f, '  <component %s />' % (file.get_params())
+            print >> f, '  <component %s />' % (file.getParams())
         print >> f, '</package>'
         f.close()
         
@@ -132,46 +124,41 @@ class PackageMaker:
         """ Remove all the files in the named directory.  Does not
         operate recursively. """
 
-        for filename in os.listdir(dirname):
-            pathname = os.path.join(dirname, filename)
-            try:
-                os.unlink(pathname)
-            except OSError:
-                pass
+        for filename in os.listdir(dirname.toOsSpecific()):
+            pathname = Filename(dirname, filename)
+            pathname.unlink()
 
     def addComponents(self):
         """ Walks through all the files in the start directory and
         adds them to the archive.  Recursively visits
         sub-directories. """
 
-        startDir = self.startDir
+        startDir = self.startDir.toOsSpecific()
         if startDir.endswith(os.sep):
             startDir = startDir[:-1]
-        elif os.altsep and startDir.endswith(os.altsep):
-            startDir = startDir[:-1]
         prefix = startDir + os.sep
         for dirpath, dirnames, filenames in os.walk(startDir):
             if dirpath == startDir:
                 localpath = ''
             else:
                 assert dirpath.startswith(prefix)
-                localpath = dirpath[len(prefix):]
+                localpath = dirpath[len(prefix):] + '/'
 
             for basename in filenames:
-                file = FileSpec(os.path.join(localpath, basename),
-                                os.path.join(startDir, basename))
+                file = FileSpec(localpath + basename,
+                                Filename(self.startDir, basename))
                 print file.filename
                 self.components.append(file)
-                self.archive.add(file.pathname, file.filename, recursive = False)
+                self.archive.addSubfile(file.filename, file.pathname, 0)
                 
 def makePackage(args):
     opts, args = getopt.getopt(args, 'd:p:v:h')
 
     pm = PackageMaker()
-    pm.startDir = '.'
+    pm.startDir = Filename('.')
     for option, value in opts:
         if option == '-d':
-            pm.stageDir = Filename.fromOsSpecific(value).toOsSpecific()
+            pm.stageDir = Filename.fromOsSpecific(value)
         elif option == '-p':
             pm.packageName = value
         elif option == '-v':

direct/src/plugin/p3dMultifileReader.I

@@ -0,0 +1,40 @@
+// Filename: p3dMultifileReader.I
+// Created by:  drose (15Jun09)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DMultifileReader::read_uint16
+//       Access: Private
+//  Description: Extracts an unsigned short from the file.
+////////////////////////////////////////////////////////////////////
+inline unsigned int P3DMultifileReader::
+read_uint16() {
+  unsigned int a = _in.get();
+  unsigned int b = _in.get();
+  return (b << 8) | a;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DMultifileReader::read_uint32
+//       Access: Private
+//  Description: Extracts an unsigned long from the file.
+////////////////////////////////////////////////////////////////////
+inline unsigned int P3DMultifileReader::
+read_uint32() {
+  unsigned int a = _in.get();
+  unsigned int b = _in.get();
+  unsigned int c = _in.get();
+  unsigned int d = _in.get();
+  return (d << 24) | (c << 16) | (b << 8) | a;
+}

direct/src/plugin/p3dMultifileReader.cxx

@@ -0,0 +1,162 @@
+// Filename: p3dMultifileReader.cxx
+// Created by:  drose (15Jun09)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "p3dMultifileReader.h"
+
+// This sequence of bytes begins each Multifile to identify it as a
+// Multifile.
+const char P3DMultifileReader::_header[] = "pmf\0\n\r";
+const size_t P3DMultifileReader::_header_size = 6;
+
+const int P3DMultifileReader::_current_major_ver = 1;
+const int P3DMultifileReader::_current_minor_ver = 1;
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DMultifileReader::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+P3DMultifileReader::
+P3DMultifileReader() {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DMultifileReader::extract
+//       Access: Public
+//  Description: Reads the named multifile, and extracts all files
+//               within it to the indicated directory.  Returns true
+//               on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool P3DMultifileReader::
+extract(const string &pathname, const string &to_dir) {
+  _subfiles.clear();
+
+  _in.open(pathname.c_str(), ios::in | ios::binary);
+  if (!_in) {
+    cerr << "Couldn't open " << pathname << "\n";
+    return false;
+  }
+
+  for (size_t i = 0; i < _header_size; ++i) {
+    int ch = _in.get();
+    if (ch != _header[i]) {
+      cerr << "Failed header check: " << pathname << "\n";
+      return false;
+    }
+  }
+
+  unsigned int major = read_uint16();
+  unsigned int minor = read_uint16();
+  if (major != _current_major_ver || minor != _current_minor_ver) {
+    cerr << "Incompatible multifile version: " << pathname << "\n";
+    return false;
+  }
+
+  unsigned int scale = read_uint32();
+  if (scale != 1) {
+    cerr << "Unsupported scale factor in " << pathname << "\n";
+    return false;
+  }
+
+  // We don't care about the timestamp.
+  read_uint32();
+
+  if (!read_index()) {
+    cerr << "Error reading multifile index\n";
+    return false;
+  }
+
+  // Now walk through all of the files.
+  Subfiles::iterator si;
+  for (si = _subfiles.begin(); si != _subfiles.end(); ++si) {
+    const Subfile &s = (*si);
+    cerr << s._filename << "\n";
+
+    string output_pathname = to_dir + "/" + s._filename;
+    ofstream out(output_pathname.c_str(), ios::out | ios::trunc | ios::binary);
+    if (!out) {
+      cerr << "Unable to create " << output_pathname << "\n";
+      return false;
+    }
+
+    _in.seekg(s._start);
+
+    static const size_t buffer_size = 1024;
+    char buffer[buffer_size];
+
+    size_t remaining_data = s._length;
+    _in.read(buffer, min(buffer_size, remaining_data));
+    size_t count = _in.gcount();
+    while (count != 0) {
+      remaining_data -= count;
+      out.write(buffer, count);
+      _in.read(buffer, min(buffer_size, remaining_data));
+      count = _in.gcount();
+    }
+
+    if (remaining_data != 0) {
+      cerr << "Unable to extract " << s._filename << "\n";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DMultifileReader::read_index
+//       Access: Public
+//  Description: Assuming the file stream is positioned at the first
+//               record, reads all of the records into the _subfiles
+//               list.  Returns true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool P3DMultifileReader::
+read_index() {
+  unsigned int next_entry = read_uint32();
+  if (!_in) {
+    return false;
+  }
+  while (next_entry != 0) {
+    Subfile s;
+    s._start = read_uint32();
+    s._length = read_uint32();
+    unsigned int flags = read_uint16();
+    if (flags != 0) {
+      cerr << "Unsupported per-subfile options in multifile\n";
+      return false;
+    }
+    read_uint32();
+    size_t name_length = read_uint16();
+    char *buffer = new char[name_length];
+    _in.read(buffer, name_length);
+
+    // The filenames are xored with 0xff just for fun.
+    for (size_t ni = 0; ni < name_length; ++ni) {
+      buffer[ni] ^= 0xff;
+    }
+
+    s._filename = string(buffer, name_length);
+    delete[] buffer;
+
+    _subfiles.push_back(s);
+
+    _in.seekg(next_entry);
+    next_entry = read_uint32();
+    if (!_in) {
+      return false;
+    }
+  }
+
+  return true;
+}

direct/src/plugin/p3dMultifileReader.h

@@ -0,0 +1,59 @@
+// Filename: p3dMultifileReader.h
+// Created by:  drose (15Jun09)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef P3DMULTIFILEREADER_H
+#define P3DMULTIFILEREADER_H
+
+#include "p3d_plugin_common.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : P3DMultifileReader
+// Description : A way-simple implementation of Panda's multifile
+//               reader.  See panda/src/express/multifile.cxx for a
+//               full description of the binary format.  This
+//               implementation doesn't support per-subfile
+//               compression or encryption.
+////////////////////////////////////////////////////////////////////
+class P3DMultifileReader {
+public:
+  P3DMultifileReader();
+
+  bool extract(const string &pathname, const string &to_dir);
+
+private:
+  bool read_index();
+  inline unsigned int read_uint16();
+  inline unsigned int read_uint32();
+
+  ifstream _in;
+
+  class Subfile {
+  public:
+    string _filename;
+    size_t _start;
+    size_t _length;
+  };
+
+  typedef vector<Subfile> Subfiles;
+  Subfiles _subfiles;
+
+  static const char _header[];
+  static const size_t _header_size;
+  static const int _current_major_ver;
+  static const int _current_minor_ver;
+};
+
+#include "p3dMultifileReader.I"
+
+#endif

direct/src/plugin/p3dPackage.I

@@ -47,6 +47,26 @@ get_package_dir() const {
   return _package_dir;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_package_name
+//       Access: Public
+//  Description: Returns the name of this package.
+////////////////////////////////////////////////////////////////////
+inline const string &P3DPackage::
+get_package_name() const {
+  return _package_name;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_package_version
+//       Access: Public
+//  Description: Returns the version string of this package.
+////////////////////////////////////////////////////////////////////
+inline const string &P3DPackage::
+get_package_version() const {
+  return _package_version;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::decode_hexdigit
 //       Access: Private

direct/src/plugin/p3dPackage.cxx

@@ -14,10 +14,10 @@
 
 #include "p3dPackage.h"
 #include "p3dInstanceManager.h"
+#include "p3dMultifileReader.h"
 
 #include "openssl/md5.h"
 #include "zlib.h"
-#include "libtar.h"
 
 #include <algorithm>
 #include <fstream>
@@ -301,8 +301,8 @@ uncompress_archive() {
   string source_pathname = _package_dir + "/" + _compressed_archive._filename;
   string target_pathname = _package_dir + "/" + _uncompressed_archive._filename;
 
-  gzFile source = gzopen(source_pathname.c_str(), "rb");
-  if (source == NULL) {
+  ifstream source(source_pathname.c_str(), ios::in | ios::binary);
+  if (!source) {
     cerr << "Couldn't open " << source_pathname << "\n";
     report_done(false);
     return;
@@ -312,38 +312,85 @@ uncompress_archive() {
   if (!target) {
     cerr << "Couldn't write to " << target_pathname << "\n";
     report_done(false);
+    return;
   }
 
-  static const int buffer_size = 1024;
-  char buffer[buffer_size];
-
-  int count = gzread(source, buffer, buffer_size);
-  while (count > 0) {
-    target.write(buffer, count);
-    count = gzread(source, buffer, buffer_size);
-  }
+  z_stream z;
+  z.next_in = Z_NULL;
+  z.avail_in = 0;
+  z.zalloc = Z_NULL;
+  z.zfree = Z_NULL;
+  z.opaque = Z_NULL;
+  z.msg = (char *)"no error message";
 
-  if (count < 0) {
-    cerr << "gzip error decompressing " << source_pathname << "\n";
-    int errnum;
-    cerr << gzerror(source, &errnum) << "\n";
-    gzclose(source);
+  int result = inflateInit(&z);
+  if (result < 0) {
+    cerr << z.msg << "\n";
     report_done(false);
     return;
   }
+  
+  static const int decompress_buffer_size = 1024;
+  char decompress_buffer[decompress_buffer_size];
+  static const int write_buffer_size = 1024;
+  char write_buffer[write_buffer_size];
+
+  bool eof = false;
+  int flush = 0;
+
+  while (true) {
+    if (z.avail_in == 0 && !eof) {
+      source.read(decompress_buffer, decompress_buffer_size);
+      size_t read_count = source.gcount();
+      eof = (read_count == 0 || source.eof() || source.fail());
+        
+      z.next_in = (Bytef *)decompress_buffer;
+      z.avail_in = read_count;
+    }
 
-  gzclose(source);
-    
-  if (!target) {
-    cerr << "Couldn't write entire file to " << target_pathname << "\n";
+    z.next_out = (Bytef *)write_buffer;
+    z.avail_out = write_buffer_size;
+    int result = inflate(&z, flush);
+    if (z.avail_out < write_buffer_size) {
+      target.write(write_buffer, write_buffer_size - z.avail_out);
+      if (!target) {
+        cerr << "Couldn't write entire file to " << target_pathname << "\n";
+        report_done(false);
+        return;
+      }
+    }
+
+    if (result == Z_STREAM_END) {
+      // Here's the end of the file.
+      break;
+
+    } else if (result == Z_BUF_ERROR && flush == 0) {
+      // We might get this if no progress is possible, for instance if
+      // the input stream is truncated.  In this case, tell zlib to
+      // dump everything it's got.
+      flush = Z_FINISH;
+
+    } else if (result < 0) {
+      cerr << z.msg << "\n";
+      inflateEnd(&z);
+      report_done(false);
+      return;
+    }
+  }
+
+  result = inflateEnd(&z);
+  if (result < 0) {
+    cerr << z.msg << "\n";
     report_done(false);
     return;
   }
 
+  source.close();
   target.close();
 
   if (!_uncompressed_archive.full_verify(_package_dir)) {
-    cerr << "after uncompressing " << target_pathname << ", failed hash check\n";
+    cerr << "after uncompressing " << target_pathname
+         << ", failed hash check\n";
     report_done(false);
     return;
   }
@@ -364,25 +411,14 @@ extract_archive() {
   cerr << "extracting " << _uncompressed_archive._filename << "\n";
 
   string source_pathname = _package_dir + "/" + _uncompressed_archive._filename;
-
-  TAR *tar = NULL;
-  int result = tar_open
-    (&tar, (char *)source_pathname.c_str(), NULL, O_RDONLY, 0666, TAR_VERBOSE);
-  if (result != 0) {
-    cerr << "Unable to open " << source_pathname << "\n";
+  P3DMultifileReader reader;
+  if (!reader.extract(source_pathname, _package_dir)) {
+    cerr << "Failure extracting " << _uncompressed_archive._filename
+         << "\n";
     report_done(false);
     return;
   }
 
-  while (th_read(tar) == 0) {
-    string basename = th_get_pathname(tar);
-    cerr << basename << "\n";
-    string pathname = _package_dir + "/" + basename;
-    tar_extract_file(tar, (char *)pathname.c_str());
-  }
-
-  tar_close(tar);
-
   cerr << "done extracting\n";
   report_done(true);
 }

direct/src/plugin/p3dPackage.h

@@ -43,6 +43,8 @@ public:
   inline bool get_ready() const;
   inline bool get_failed() const;
   inline const string &get_package_dir() const;
+  inline const string &get_package_name() const;
+  inline const string &get_package_version() const;
 
   void set_callback(Callback *callback);
   void cancel_callback(Callback *callback);

direct/src/plugin/p3dSession.cxx

@@ -649,7 +649,12 @@ package_ready(P3DPackage *package, bool success) {
   if (this == _session->_panda3d_callback) {
     _session->_panda3d_callback = NULL;
     if (package == _session->_panda3d) {
-      _session->start_p3dpython();
+      if (success) {
+        _session->start_p3dpython();
+      } else {
+        cerr << "Failed to install " << package->get_package_name()
+             << "_" << package->get_package_version() << "\n";
+      }
     } else {
       cerr << "Unexpected panda3d package: " << package << "\n";
     }

direct/src/plugin/p3d_plugin_composite1.cxx

@@ -3,5 +3,6 @@
 #include "p3dFileDownload.cxx"
 #include "p3dInstance.cxx"
 #include "p3dInstanceManager.cxx"
+#include "p3dMultifileReader.cxx"
 #include "p3dPackage.cxx"
 #include "p3dSession.cxx"