use _vfsimporter.pyd instead of runp3d_frozen.pyd

direct/src/p3d/AppRunner.py

@@ -14,7 +14,20 @@ import os
 import types
 import __builtin__
 
-from direct.showbase import VFSImporter
+if 'VFSImporter' in sys.modules:
+    # If we've already got a VFSImporter module defined at the
+    # toplevel, we must have come in here by way of the
+    # p3dPythonRun.cxx program, which starts out by importing a frozen
+    # VFSImporter.  Let's make sure we don't have two VFSImporter
+    # modules.
+    import VFSImporter
+    import direct.showbase
+    direct.showbase.VFSImporter = VFSImporter
+    sys.modules['direct.showbase.VFSImporter'] = VFSImporter
+else:
+    # Otherwise, we can import the VFSImporter normally.
+    from direct.showbase import VFSImporter
+
 from direct.showbase.DirectObject import DirectObject
 from pandac.PandaModules import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, HTTPClient, Thread, WindowProperties, readXmlStream, ExecutionEnvironment, PandaSystem, URLSpec
 from direct.stdpy import file
@@ -405,7 +418,6 @@ class AppRunner(DirectObject):
 
         # Mount the Multifile under /mf, by convention.
         vfs.mount(mf, self.multifileRoot, vfs.MFReadOnly)
-        VFSImporter.freeze_new_modules(mf, self.multifileRoot)
 
         self.loadMultifilePrcFiles(mf, self.multifileRoot)
         self.gotP3DFilename = True

direct/src/p3d/PackageInfo.py

@@ -1,5 +1,6 @@
 from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, TiXmlDocument, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread
 from direct.p3d.FileSpec import FileSpec
+from direct.showbase import VFSImporter
 import os
 import sys
 
@@ -343,8 +344,22 @@ class PackageInfo:
 
         appRunner.loadMultifilePrcFiles(mf, root)
 
-        if root not in sys.path:
+        # Add this to the Python search path, if it's not already
+        # there.  We have to take a bit of care to check if it's
+        # already there, since there can be some ambiguity in
+        # os-specific path strings.
+        root = self.packageDir.toOsSpecific()
+        foundOnPath = False
+        for p in sys.path:
+            if root == p:
+                # Already here, exactly.
+                foundOnPath = True
+                break
+            elif root == Filename.fromOsSpecific(p).toOsSpecific():
+                # Already here, with some futzing.
+                foundOnPath = True
+                break
+
+        if not foundOnPath:
+            # Not already here; add it.
             sys.path.append(root)
-        #print "Installed %s %s" % (self.packageName, self.packageVersion)
-        
-        

direct/src/p3d/Packager.py

@@ -126,7 +126,7 @@ class Packager:
                 
     class ExcludeFilename:
         def __init__(self, filename, caseSensitive):
-            self.localOnly = (not filename.get_dirname())
+            self.localOnly = (not filename.getDirname())
             if not self.localOnly:
                 filename = Filename(filename)
                 filename.makeCanonical()
@@ -2089,23 +2089,19 @@ class Packager:
 
         self.currentPackage.requirePackage(package)
 
-    def do_module(self, *args):
+    def do_module(self, *args, **kw):
         """ Adds the indicated Python module(s) to the current package. """
+        self.addModule(args, **kw)
 
+    def addModule(self, moduleNames, newName = None, filename = None):
         if not self.currentPackage:
             raise OutsideOfPackageError
 
-        for moduleName in args:
-            self.currentPackage.freezer.addModule(moduleName)
-
-    def do_renameModule(self, moduleName, newName):
-        """ Adds the indicated Python module to the current package,
-        renaming to a new name. """
-
-        if not self.currentPackage:
-            raise OutsideOfPackageError
+        if (newName or filename) and len(moduleNames) != 1:
+            raise PackagerError, 'Cannot specify newName with multiple modules'
 
-        self.currentPackage.freezer.addModule(moduleName, newName = newName)
+        for moduleName in moduleNames:
+            self.currentPackage.freezer.addModule(moduleName, newName = newName, filename = filename)
 
     def do_excludeModule(self, *args):
         """ Marks the indicated Python module as not to be included. """
@@ -2139,6 +2135,45 @@ class Packager:
 
         self.currentPackage.mainModule = (moduleName, newName)
 
+    def do_setupPanda3D(self):
+        """ A special convenience command that adds the minimum
+        startup modules for a panda3d package, intended for developers
+        producing their own custom panda3d for download.  Should be
+        called before any other Python modules are named. """
+
+        # First, freeze just VFSImporter.py into its own
+        # _vfsimporter.pyd file.  This one is a special case, because
+        # we need this code in order to load python files from the
+        # Multifile, so this file can't itself be in the Multifile.
+
+        # This requires a bit of care, because we only want to freeze
+        # VFSImporter.py, and not any other part of direct.
+        self.do_excludeModule('direct')
+
+        # Import the actual VFSImporter module to get its filename on
+        # disk.
+        from direct.showbase import VFSImporter
+        filename = Filename.fromOsSpecific(VFSImporter.__file__)
+        
+        self.do_module('VFSImporter', filename = filename)
+        self.do_freeze('_vfsimporter', compileToExe = False)
+
+        # Now that we're done freezing, explicitly add 'direct' to
+        # counteract the previous explicit excludeModule().
+        self.do_module('direct')
+
+        # This is the key Python module that is imported at runtime to
+        # start an application running.
+        self.do_module('direct.p3d.AppRunner')
+
+        # This is the main program that drives the runtime Python.  It
+        # is responsible for loading _vfsimporter.pyd, and then
+        # importing direct.p3d.AppRunner, to start an application
+        # running.  Note that the .exe extension is automatically
+        # replaced with the platform-specific extension appropriate
+        # for an executable.
+        self.do_file('p3dpython.exe')
+
     def do_freeze(self, filename, compileToExe = False):
         """ Freezes all of the current Python code into either an
         executable (if compileToExe is true) or a dynamic library (if
@@ -2317,6 +2352,7 @@ class Packager:
         if not self.currentPackage:
             raise OutsideOfPackageError
 
+        filename = Filename(filename)
         self.currentPackage.excludeFile(filename)
 
     def do_dir(self, dirname, newDir = None, unprocessed = None):

direct/src/p3d/panda3d.pdef

@@ -34,11 +34,12 @@ class panda3d(package):
 
     config(display_name = "Panda3D")
 
-    # This is the key Python module that is imported at runtime to start
-    # an application running.
-    module('direct.p3d.AppRunner')
-
-    # These are additional Python modules that are needed by most Panda3D
+    # First, add the minimum startup files for a Panda3D package.
+    # These are files that the Panda3D runtime will explicitly look
+    # for by name in order to get itself bootstrapped.
+    setupPanda3D()
+    
+    # These are Python modules that are needed by most Panda3D
     # applications.  It doesn't matter too much if we miss one or two
     # here, since any module imported by any of this code will
     # automatically be included as well, and we end up with a pretty
@@ -70,23 +71,6 @@ class panda3d(package):
                   'direct.tkpanels',
                   'direct.tkwidgets')
 
-    # Bind all of the above Python code into a frozen DLL.  This makes the
-    # Python code available when the DLL is imported.  It is actually
-    # preferable not to use freeze, but instead just to leave the Python
-    # code directly within the Multifile; but in this case we have to use
-    # freeze on this very first package, due to bootstrapping
-    # requirements.  (Part of the code we're including here is the code
-    # required to load Python code from a Multifile, so it can't be placed
-    # within a Multifile itself.)
-    freeze('runp3d_frozen', compileToExe = False)
-
-    # This is the main program that drives the plugin application.  It is
-    # responsible for loading runp3d_frozen, above, and then importing
-    # direct.p3d.runp3d, to start an application running.  Note that
-    # the .exe extension is automatically replaced with the
-    # platform-specific extension appropriate for an executable.
-    file('p3dpython.exe')
-
     # Most of the core Panda3D DLL's will be included implicitly due to
     # being referenced by the above Python code.  Here we name a few more
     # that are also needed, but aren't referenced by any code.  Again,
@@ -117,58 +101,58 @@ default-model-extension .bam
 """ + auxDisplays)
 
 
-class egg(package):
-    # This package contains the code for reading and operating on egg
-    # files.  Since the Packager automatically converts egg files to bam
-    # files, this is not needed for most Panda3D applications.
+## class egg(package):
+##     # This package contains the code for reading and operating on egg
+##     # files.  Since the Packager automatically converts egg files to bam
+##     # files, this is not needed for most Panda3D applications.
 
-    config(display_name = "Panda3D egg loader")
-    require('panda3d')
+##     config(display_name = "Panda3D egg loader")
+##     require('panda3d')
 
-    file('libpandaegg.dll')
+##     file('libpandaegg.dll')
 
-    file('egg.prc', extract = True, text = """
-plugin-path $EGG_ROOT
-load-file-type egg pandaegg
-""")
+##     file('egg.prc', extract = True, text = """
+## plugin-path $EGG_ROOT
+## load-file-type egg pandaegg
+## """)
 
-class wx(package):
-    config(display_name = "wxPython GUI Toolkit")
-    require('panda3d')
+## class wx(package):
+##     config(display_name = "wxPython GUI Toolkit")
+##     require('panda3d')
 
-    module('direct.showbase.WxGlobal', 'wx', 'wx.*')
+##     module('direct.showbase.WxGlobal', 'wx', 'wx.*')
 
 
-class tk(package):
-    config(display_name = "Tk GUI Toolkit")
-    require('panda3d')
+## class tk(package):
+##     config(display_name = "Tk GUI Toolkit")
+##     require('panda3d')
 
-    module('Tkinter',
-           'direct.showbase.TkGlobal',
-           'direct.tkpanels',
-           'direct.tkwidgets')
+##     module('Tkinter',
+##            'direct.showbase.TkGlobal',
+##            'direct.tkpanels',
+##            'direct.tkwidgets')
 
-class packp3d(p3d):
-    # This application is a command-line convenience for building a p3d
-    # application out of a directory hierarchy on disk.  We build it here
-    # into its own p3d application, to allow end-users to easily build p3d
-    # applications using the appropriate version of Python and Panda for
-    # the targeted runtime.
+## class packp3d(p3d):
+##     # This application is a command-line convenience for building a p3d
+##     # application out of a directory hierarchy on disk.  We build it here
+##     # into its own p3d application, to allow end-users to easily build p3d
+##     # applications using the appropriate version of Python and Panda for
+##     # the targeted runtime.
 
-    config(display_name = "Panda3D Application Packer",
-           hidden = True, platform_specific = False)
-    require('panda3d', 'egg')
+##     config(display_name = "Panda3D Application Packer",
+##            hidden = True, platform_specific = False)
+##     require('panda3d', 'egg')
 
-    mainModule('direct.p3d.packp3d')
+##     mainModule('direct.p3d.packp3d')
 
 
-class ppackage(p3d):
-    # As above, a packaging utility.  This is the fully-general ppackage
-    # utility, which reads pdef files (like this one!) and creates one or
-    # more packages or p3d applications.
+## class ppackage(p3d):
+##     # As above, a packaging utility.  This is the fully-general ppackage
+##     # utility, which reads pdef files (like this one!) and creates one or
+##     # more packages or p3d applications.
 
-    config(display_name = "Panda3D General Package Utility",
-           hidden = True, platform_specific = False)
-    require('panda3d', 'egg')
+##     config(display_name = "Panda3D General Package Utility",
+##            hidden = True, platform_specific = False)
+##     require('panda3d', 'egg')
 
-    mainModule('direct.p3d.ppackage')
+##     mainModule('direct.p3d.ppackage')

direct/src/plugin/p3dHost.cxx

@@ -378,6 +378,11 @@ copy_file(const string &from_filename, const string &to_filename) {
     return true;
   }
 
+  unlink(to_filename.c_str());
+  if (rename(temp_filename.c_str(), to_filename.c_str()) == 0) {
+    return true;
+  }
+
   unlink(temp_filename.c_str());
   return false;
 }

direct/src/plugin/p3dPackage.I

@@ -152,3 +152,15 @@ get_desc_file_pathname() const {
   return _desc_file_pathname;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::get_archive_file_pathname
+//       Access: Public
+//  Description: Returns the full path to the package's uncompressed
+//               archive file.  This is only valid if get_ready() is
+//               true and the package is not a "solo" package.
+////////////////////////////////////////////////////////////////////
+inline string P3DPackage::
+get_archive_file_pathname() const {
+  return _uncompressed_archive.get_pathname(_package_dir);
+}
+

direct/src/plugin/p3dPackage.h

@@ -58,6 +58,7 @@ public:
   inline const string &get_package_display_name() const;
 
   inline const string &get_desc_file_pathname() const;
+  inline string get_archive_file_pathname() const;
 
   void add_instance(P3DInstance *inst);
   void remove_instance(P3DInstance *inst);

direct/src/plugin/p3dPythonRun.cxx

@@ -15,6 +15,8 @@
 #include "p3dPythonRun.h"
 #include "asyncTaskManager.h"
 #include "binaryXml.h"
+#include "multifile.h"
+#include "virtualFileSystem.h"
 
 // There is only one P3DPythonRun object in any given process space.
 // Makes the statics easier to deal with, and we don't need multiple
@@ -36,10 +38,20 @@ P3DPythonRun(int argc, char *argv[]) {
   _session_id = 0;
   _next_sent_id = 0;
 
-  _program_name = argv[0];
+  if (argc >= 1) {
+    _program_name = argv[0];
+  }
+  if (argc >= 2) {
+    _archive_file = Filename::from_os_specific(argv[1]);
+  }
+  if (_archive_file.empty()) {
+    nout << "No archive filename specified on command line.\n";
+    exit(1);
+  }
+
   _py_argc = 1;
   _py_argv = (char **)malloc(2 * sizeof(char *));
-  _py_argv[0] = argv[0];
+  _py_argv[0] = (char *)_program_name.c_str();
   _py_argv[1] = NULL;
 
 #ifdef NDEBUG
@@ -116,17 +128,49 @@ run_python() {
   
 #endif
 
-  // First, load runp3d_frozen.pyd.  Since this is a magic frozen pyd,
+  // First, load _vfsimporter.pyd.  Since this is a magic frozen pyd,
   // importing it automatically makes all of its frozen contents
   // available to import as well.
-  PyObject *runp3d_frozen = PyImport_ImportModule("runp3d_frozen");
-  if (runp3d_frozen == NULL) {
+  PyObject *vfsimporter = PyImport_ImportModule("_vfsimporter");
+  if (vfsimporter == NULL) {
+    PyErr_Print();
+    return false;
+  }
+  Py_DECREF(vfsimporter);
+
+  // And now we can import the VFSImporter module that was so defined.
+  PyObject *vfsimporter_module = PyImport_ImportModule("VFSImporter");
+  if (vfsimporter_module == NULL) {
+    PyErr_Print();
+    return false;
+  }
+
+  // And register the VFSImporter.
+  PyObject *result = PyObject_CallMethod(vfsimporter_module, (char *)"register", (char *)"");
+  if (result == NULL) {
     PyErr_Print();
     return false;
   }
-  Py_DECREF(runp3d_frozen);
+  Py_DECREF(result);
+  Py_DECREF(vfsimporter_module);
+
+  // Now, the VFSImporter has been registered, which means we can
+  // start importing the rest of the Python modules, where are all
+  // defined in the multifile.  First, we need to mount the multifile
+  // into the VFS.
+  PT(Multifile) mf = new Multifile;
+  if (!mf->open_read(_archive_file)) {
+    nout << "Could not read " << _archive_file << "\n";
+    return false;
+  }
+  Filename dir = _archive_file.get_dirname();
+  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  if (!vfs->mount(mf, dir, VirtualFileSystem::MF_read_only)) {
+    nout << "Could not mount " << _archive_file << "\n";
+    return false;
+  }
 
-  // So now we can import the module itself.
+  // And finally, we can import the startup module.
   PyObject *app_runner_module = PyImport_ImportModule("direct.p3d.AppRunner");
   if (app_runner_module == NULL) {
     PyErr_Print();
@@ -207,7 +251,7 @@ run_python() {
 
   // Now pass that func pointer back to our AppRunner instance, so it
   // can call up to us.
-  PyObject *result = PyObject_CallMethod(_runner, (char *)"setRequestFunc", (char *)"O", request_func);
+  result = PyObject_CallMethod(_runner, (char *)"setRequestFunc", (char *)"O", request_func);
   if (result == NULL) {
     PyErr_Print();
     return false;

direct/src/plugin/p3dPythonRun.h

@@ -30,6 +30,7 @@
 #include "pdeque.h"
 #include "pmutex.h"
 #include "get_tinyxml.h"
+#include "filename.h"
 
 #include <Python.h>
 
@@ -111,6 +112,7 @@ private:
   int _session_id;
 
   string _program_name;
+  Filename _archive_file;
   int _py_argc;
   char **_py_argv;
 

direct/src/plugin/p3dSession.cxx

@@ -798,15 +798,17 @@ start_p3dpython(P3DInstance *inst) {
     _log_pathname += ".log";
   }
 
+  string archive_file = inst->_panda3d->get_archive_file_pathname();
+
   nout << "Attempting to start python from " << p3dpython << "\n";
 #ifdef _WIN32
   _p3dpython_handle = win_create_process
-    (p3dpython, start_dir, env, _log_pathname,
+    (p3dpython, archive_file, start_dir, env, _log_pathname,
      _pipe_read, _pipe_write);
   bool started_p3dpython = (_p3dpython_handle != INVALID_HANDLE_VALUE);
 #else
   _p3dpython_pid = posix_create_process
-    (p3dpython, start_dir, env, _log_pathname,
+    (p3dpython, archive_file, start_dir, env, _log_pathname,
      _pipe_read, _pipe_write);
   bool started_p3dpython = (_p3dpython_pid > 0);
 #endif
@@ -989,7 +991,8 @@ rt_terminate() {
 //               or INVALID_HANDLE_VALUE on falure.
 ////////////////////////////////////////////////////////////////////
 HANDLE P3DSession::
-win_create_process(const string &program, const string &start_dir,
+win_create_process(const string &program, const string &archive_file,
+                   const string &start_dir,
                    const string &env, const string &log_pathname,
                    HandleStream &pipe_read, HandleStream &pipe_write) {
 
@@ -1055,13 +1058,25 @@ win_create_process(const string &program, const string &start_dir,
     start_dir_cstr = start_dir.c_str();
   }
 
+  ostringstream stream;
+  stream << "\"" << program << "\" \"" << archive_file << "\"";
+
+  // I'm not sure why CreateProcess wants a non-const char pointer for
+  // the command-line argument, but I'm not taking chances.  It gets a
+  // non-const char array that it can modify.
+  string command_line_str = stream.str();
+  char *command_line = new char[command_line_str.size() + 1];
+  strcpy(command_line, command_line_str.c_str());
+
   PROCESS_INFORMATION process_info; 
   BOOL result = CreateProcess
-    (program.c_str(), NULL, NULL, NULL, TRUE, 0,
+    (program.c_str(), command_line, NULL, NULL, TRUE, 0,
      (void *)env.c_str(), start_dir_cstr,
      &startup_info, &process_info);
   bool started_program = (result != 0);
 
+  delete[] command_line;
+
   // Close the pipe handles that are now owned by the child.
   CloseHandle(w_from);
   CloseHandle(r_to);
@@ -1100,7 +1115,8 @@ win_create_process(const string &program, const string &start_dir,
 //               -1 on falure.
 ////////////////////////////////////////////////////////////////////
 int P3DSession::
-posix_create_process(const string &program, const string &start_dir,
+posix_create_process(const string &program, const string &archive_file,
+                     const string &start_dir,
                      const string &env, const string &log_pathname,
                      HandleStream &pipe_read, HandleStream &pipe_write) {
   // Create a bi-directional pipe to communicate with the sub-process.
@@ -1165,7 +1181,9 @@ posix_create_process(const string &program, const string &start_dir,
     }
     ptrs.push_back((char *)NULL);
     
-    execle(program.c_str(), program.c_str(), (char *)0, &ptrs[0]);
+    execle(program.c_str(), 
+           program.c_str(), archive_file.c_str(), (char *)0, 
+           &ptrs[0]);
     nout << "Failed to exec " << program << "\n";
     _exit(1);
   }

direct/src/plugin/p3dSession.h

@@ -76,12 +76,14 @@ private:
 
 #ifdef _WIN32
   static HANDLE 
-  win_create_process(const string &program, const string &start_dir,
+  win_create_process(const string &program, const string &archive_file,
+                     const string &start_dir,
                      const string &env, const string &output_filename,
                      HandleStream &pipe_read, HandleStream &pipe_write);
 #else
   static int 
-  posix_create_process(const string &program, const string &start_dir,
+  posix_create_process(const string &program, const string &archive_file,
+                       const string &start_dir,
                        const string &env, const string &output_filename,
                        HandleStream &pipe_read, HandleStream &pipe_write);
 #endif

direct/src/showbase/VFSImporter.py

@@ -1,5 +1,4 @@
-from direct.stdpy.file import open
-from pandac.PandaModules import Filename, VirtualFileSystem, VirtualFileMountSystem
+from libpandaexpress import Filename, VirtualFileSystem, VirtualFileMountSystem
 import sys
 import new
 import os
@@ -15,7 +14,8 @@ vfs = VirtualFileSystem.getGlobalPtr()
 # Possible file types.
 FTPythonSource = 0
 FTPythonCompiled = 1
-FTCompiledModule = 2
+FTExtensionModule = 2
+FTFrozenModule = 3
 
 compiledExtensions = [ 'pyc', 'pyo' ]
 if not __debug__:
@@ -32,16 +32,21 @@ class VFSImporter:
     def __init__(self, path):
         self.dir_path = Filename.fromOsSpecific(path)
 
-    def find_module(self, fullname):
+    def find_module(self, fullname, path = None):
+        if path is None:
+            dir_path = self.dir_path
+        else:
+            dir_path = path
+        #print >>sys.stderr, "find_module(%s), dir_path = %s" % (fullname, dir_path)
         basename = fullname.split('.')[-1]
-        path = Filename(self.dir_path, basename)
+        path = Filename(dir_path, basename)
 
         # First, look for Python files.
         filename = Filename(path)
         filename.setExtension('py')
         vfile = vfs.getFile(filename, True)
         if vfile:
-            return VFSLoader(self, vfile, filename, FTPythonSource)
+            return VFSLoader(dir_path, vfile, filename, FTPythonSource)
 
         # If there's no .py file, but there's a .pyc file, load that
         # anyway.
@@ -50,9 +55,9 @@ class VFSImporter:
             filename.setExtension(ext)
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(self, vfile, filename, FTPythonCompiled)
+                return VFSLoader(dir_path, vfile, filename, FTPythonCompiled)
 
-        # Look for a compiled C/C++ module.
+        # Look for a C/C++ extension module.
         for desc in imp.get_suffixes():
             if desc[2] != imp.C_EXTENSION:
                 continue
@@ -61,7 +66,7 @@ class VFSImporter:
             filename.setExtension(desc[0][1:])
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(self, vfile, filename, FTCompiledModule,
+                return VFSLoader(dir_path, vfile, filename, FTExtensionModule,
                                  desc = desc)
         
 
@@ -70,34 +75,39 @@ class VFSImporter:
         filename = Filename(path, '__init__.py')
         vfile = vfs.getFile(filename, True)
         if vfile:
-            return VFSLoader(self, vfile, filename, FTPythonSource,
+            return VFSLoader(dir_path, vfile, filename, FTPythonSource,
                              packagePath = path)
         for ext in compiledExtensions:
             filename = Filename(path, '__init__.' + ext)
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(self, vfile, filename, FTPythonCompiled,
+                return VFSLoader(dir_path, vfile, filename, FTPythonCompiled,
                                  packagePath = path)
 
+        #print >>sys.stderr, "not found."
         return None
 
 class VFSLoader:
     """ The second part of VFSImporter, this is created for a
     particular .py file or directory. """
     
-    def __init__(self, importer, vfile, filename, fileType,
+    def __init__(self, dir_path, vfile, filename, fileType,
                  desc = None, packagePath = None):
-        self.importer = importer
-        self.dir_path = importer.dir_path
-        self.timestamp = vfile.getTimestamp()
+        self.dir_path = dir_path
+        self.timestamp = None
+        if vfile:
+            self.timestamp = vfile.getTimestamp()
         self.filename = filename
         self.fileType = fileType
         self.desc = desc
         self.packagePath = packagePath
     
     def load_module(self, fullname):
-        if self.fileType == FTCompiledModule:
-            return self._import_compiled_module(fullname)
+        #print >>sys.stderr, "load_module(%s), dir_path = %s, filename = %s" % (fullname, self.dir_path, self.filename)
+        if self.fileType == FTFrozenModule:
+            return self._import_frozen_module(fullname)
+        if self.fileType == FTExtensionModule:
+            return self._import_extension_module(fullname)
         
         code = self._read_code()
         if not code:
@@ -114,8 +124,10 @@ class VFSLoader:
 
     def getdata(self, path):
         path = Filename(self.dir_path, Filename.fromOsSpecific(path))
-        f = open(path, 'rb')
-        return f.read()
+        vfile = vfs.getFile(path)
+        if not vfile:
+            raise IOError
+        return vfile.readFile(True)
 
     def is_package(self, fullname):
         return bool(self.packagePath)
@@ -131,21 +143,23 @@ class VFSLoader:
         available, or None if it is not.  May raise IOError. """
         
         if self.fileType == FTPythonCompiled or \
-           self.fileType == FTCompiledModule:
+           self.fileType == FTExtensionModule:
             return None
         
         filename = Filename(self.filename)
         filename.setExtension('py')
-        file = open(filename, 'rU')
-        return file.read()
+        vfile = vfs.getFile(filename)
+        if not vfile:
+            raise IOError
+        return vfile.readFile(True)
 
-    def _import_compiled_module(self, fullname):
-        """ Loads the compiled C/C++ shared object as a Python module,
-        and returns it. """
+    def _import_extension_module(self, fullname):
+        """ Loads the binary shared object as a Python module, and
+        returns it. """
 
         vfile = vfs.getFile(self.filename, False)
 
-        # We can only import a compiled module if it already exists on
+        # We can only import an extension module if it already exists on
         # disk.  This means if it's a truly virtual file that has no
         # on-disk equivalent, we have to write it to a temporary file
         # first.
@@ -153,26 +167,40 @@ class VFSLoader:
            isinstance(vfile.getMount(), VirtualFileMountSystem):
             # It's a real file.
             filename = self.filename
+        elif self.filename.exists():
+            # It's a virtual file, but it's shadowing a real file.
+            # Assume they're the same, and load the real one.
+            filename = self.filename
         else:
-            # It's a virtual file.  Dump it.
+            # It's a virtual file with no real-world existence.  Dump
+            # it to disk.  TODO: clean up this filename.
             filename = Filename.temporary('', self.filename.getBasenameWoExtension(),
                                           '.' + self.filename.getExtension(),
                                           type = Filename.TDso)
             filename.setExtension(self.filename.getExtension())
-            fin = open(vfile, 'rb')
-            fout = open(filename, 'wb')
-            data = fin.read(4096)
-            while data:
-                fout.write(data)
-                data = fin.read(4096)
-            fin.close()
-            fout.close()
+            filename.setBinary()
+            sin = vfile.openReadFile()
+            sout = OFileStream()
+            if not filename.openWrite(sout):
+                raise IOError
+            if not copyStream(sin, sout):
+                raise IOError
+            vfile.closeReadFile(sin)
+            del sout
 
         module = imp.load_module(fullname, None, filename.toOsSpecific(),
                                  self.desc)
         module.__file__ = self.filename.cStr()
         return module
-        
+
+    def _import_frozen_module(self, fullname):
+        """ Imports the frozen module without messing around with
+        searching any more. """
+        #print >>sys.stderr, "importing frozen %s" % (fullname)
+        module = imp.load_module(fullname, None, fullname,
+                                 ('', '', imp.PY_FROZEN))
+        #print >>sys.stderr, "got frozen %s" % (module)
+        return module
 
     def _read_code(self):
         """ Returns the Python compiled code object for this file, if
@@ -187,7 +215,7 @@ class VFSLoader:
                 return self._loadPyc(pycVfile, None)
             raise IOError, 'Could not read %s' % (self.filename)
 
-        elif self.fileType == FTCompiledModule:
+        elif self.fileType == FTExtensionModule:
             return None
 
         # It's a .py file (or an __init__.py file; same thing).  Read
@@ -222,16 +250,15 @@ class VFSLoader:
         Raises ValueError if there is a problem. """
         
         code = None
-        f = open(vfile, 'rb')
-        if f.read(4) == imp.get_magic():
-            t = struct.unpack('<I', f.read(4))[0]
+        data = vfile.readFile(True)
+        if data[:4] == imp.get_magic():
+            t = struct.unpack('<I', data[4:8])[0]
             if not timestamp or t == timestamp:
-                code = marshal.loads(f.read())
+                code = marshal.loads(data[8:])
             else:
                 raise ValueError, 'Timestamp wrong on %s' % (vfile)
         else:
             raise ValueError, 'Bad magic number in %s' % (vfile)
-        f.close()
         return code
         
 
@@ -248,7 +275,7 @@ class VFSLoader:
         pycFilename = Filename(filename)
         pycFilename.setExtension(compiledExtensions[0])
         try:
-            f = open(pycFilename, 'wb')
+            f = open(pycFilename.toOsSpecific(), 'wb')
         except IOError:
             pass
         else:
@@ -268,70 +295,13 @@ def register():
     already been registered, so that future Python import statements
     will vector through here (and therefore will take advantage of
     Panda's virtual file system). """
-    
+
     global _registered
     if not _registered:
         _registered = True
         sys.path_hooks.insert(0, VFSImporter)
-        
-def freeze_new_modules(multifile, root_path):
-    """ Walks the multifile and looks for Python packages that are
-    children of frozen modules.  These are converted to frozen
-    modules, since the Python runtime system only supports loading
-    frozen children of frozen modules.
-
-    The multifile must be already mounted at root_path.  """
-
-    # This module is defined by extend_frozen.c in
-    # direct/src/showbase.  It's a special extension module that
-    # provides hooks into the array of frozen modules, which is
-    # otherwise accessible only to the C level.
-    import extend_frozen
-
-    modules = []
-    pyExtensions = ['py'] + compiledExtensions
-    for filename in multifile.getSubfileNames():
-        filename = Filename(filename)
-        ext = filename.getExtension()
-        if ext in pyExtensions:
-            # A Python file.
-            moduleName = Filename(filename)
-            moduleName.setExtension('')
-            isPackage = False
-            if moduleName.getBasename() == '__init__':
-                # A package.
-                moduleName = moduleName.getDirname()
-            else:
-                moduleName = moduleName.cStr()
-            moduleName = '.'.join(moduleName.split('/'))
-            modules.append(moduleName)
 
-    modules.sort()
-    
-    # Now look for any children of frozen modules; these children need
-    # to become frozen modules themselves.
-    existingFrozenModules = {}
-    newFrozen = []
-    for moduleName in modules:
-        if extend_frozen.is_frozen_module(moduleName):
-            # It's a frozen module.  All children require freezing also.
-            existingFrozenModules[moduleName] = True
-        else:
-            # It's not a frozen module, but maybe it needs to be.
-            if '.' in moduleName:
-                parentModuleName = moduleName.rsplit('.', 1)[0]
-                if parentModuleName in existingFrozenModules:
-                    # Bad news.  We have to freeze this one.
-                    existingFrozenModules[moduleName] = True
-
-                    # Load up the module code.
-                    path = root_path + '/' + '/'.join(moduleName.split('.')[:-1])
-                    importer = VFSImporter(path)
-                    loader = importer.find_module(moduleName)
-                    if loader:
-                        code = loader.get_code(moduleName)
-                        newFrozen.append((moduleName, marshal.dumps(code)))
-
-    # Now pass our list of newly-frozen modules to the low level code.
-    if newFrozen:
-        extend_frozen.extend(newFrozen)
+        # Blow away the importer cache, so we'll come back through the
+        # VFSImporter for every folder in the future, even those
+        # folders that previously were loaded directly.
+        sys.path_importer_cache = {}

direct/src/showbase/extend_frozen.c

@@ -6,6 +6,15 @@
 #define DLLEXPORT
 #endif
 
+/*
+ * This pointer is kept internally to this module.  It represents the
+ * locally-allocated FrozenModules array.  If the
+ * PyImport_FrozenModules is any other value, then it wasn't allocated
+ * via this module.
+ */
+static struct _frozen *frozen_modules = NULL;
+static int num_frozen_modules = 0;
+
 /*
  * Call this function to extend the frozen modules array with a new
  * array of frozen modules, provided in a C-style array, at runtime.
@@ -16,10 +25,18 @@ extend_frozen_modules(const struct _frozen *new_modules, int new_count) {
   int orig_count;
   struct _frozen *realloc_FrozenModules;
 
-  /* First, count the number of frozen modules we had originally. */
-  orig_count = 0;
-  while (PyImport_FrozenModules[orig_count].name != NULL) {
-    ++orig_count;
+  if (PyImport_FrozenModules == frozen_modules) {
+    /* If the previous array was allocated through this module, we
+       already know the count. */
+    orig_count = num_frozen_modules;
+
+  } else {
+    /* If the previous array came from anywhere else, we have to count
+       up its length. */
+    orig_count = 0;
+    while (PyImport_FrozenModules[orig_count].name != NULL) {
+      ++orig_count;
+    }
   }
 
   if (new_count == 0) {
@@ -28,10 +45,16 @@ extend_frozen_modules(const struct _frozen *new_modules, int new_count) {
   }
 
   /* Reallocate the PyImport_FrozenModules array bigger to make room
-     for the additional frozen modules.  We just leak the original
-     array; it's too risky to try to free it. */
+     for the additional frozen modules. */
   realloc_FrozenModules = (struct _frozen *)malloc((orig_count + new_count + 1) * sizeof(struct _frozen));
 
+  /* If the previous array was allocated through this module, we can
+     free it; otherwise, we have to leak it. */
+  if (frozen_modules != NULL) {
+    free(frozen_modules);
+    frozen_modules = NULL;
+  }
+
   /* The new frozen modules go at the front of the list. */
   memcpy(realloc_FrozenModules, new_modules, new_count * sizeof(struct _frozen));
 
@@ -43,8 +66,10 @@ extend_frozen_modules(const struct _frozen *new_modules, int new_count) {
 
   /* Assign the new pointer. */
   PyImport_FrozenModules = realloc_FrozenModules;
+  frozen_modules = realloc_FrozenModules;
+  num_frozen_modules = orig_count + new_count;
 
-  return orig_count + new_count;
+  return num_frozen_modules;
 }
 
 /* 

direct/src/showutil/FreezeTool.py

@@ -762,6 +762,7 @@ class Freezer:
                 self.__loadModule(mdef)
             except ImportError:
                 print "Unknown module: %s" % (mdef.moduleName)
+                import pdb; pdb.set_trace()
 
         # Also attempt to import any implicit modules.  If any of
         # these fail to import, we don't really care.
@@ -823,9 +824,15 @@ class Freezer:
                 self.mf.path.append(tempPath)
 
             pathname = mdef.filename.toOsSpecific()
-            fp = open(pathname, modulefinder.READ_MODE)
-            stuff = ("", "r", imp.PY_SOURCE)
-            self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
+            ext = mdef.filename.getExtension()
+            if ext == 'pyc' or ext == 'pyo':
+                fp = open(pathname, 'rb')
+                stuff = ("", "rb", imp.PY_COMPILED)
+                self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
+            else:
+                fp = open(pathname, modulefinder.READ_MODE)
+                stuff = ("", "r", imp.PY_SOURCE)
+                self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
 
             if tempPath:
                 del self.mf.path[-1]