Python 3 support to rtdist, reimplement VFS open() on top of io.IOBase, allow freezing into .c target, make VFSImporter and friends frozen into p3dpython

direct/src/p3d/AppRunner.py

@@ -13,22 +13,9 @@ __all__ = ["AppRunner", "dummyAppRunner", "ArgumentError"]
 
 import sys
 import os
-import __builtin__
-
-if 'VFSImporter' in sys.modules:
-    # If we've already got a VFSImporter module defined at the
-    # toplevel, we must have come in here by way of the
-    # p3dPythonRun.cxx program, which starts out by importing a frozen
-    # VFSImporter.  Let's make sure we don't have two VFSImporter
-    # modules.
-    import VFSImporter
-    import direct.showbase
-    direct.showbase.VFSImporter = VFSImporter
-    sys.modules['direct.showbase.VFSImporter'] = VFSImporter
-else:
-    # Otherwise, we can import the VFSImporter normally.
-    from direct.showbase import VFSImporter
+import __builtin__ as builtins
 
+from direct.showbase import VFSImporter
 from direct.showbase.DirectObject import DirectObject
 from panda3d.core import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, WindowProperties, ExecutionEnvironment, PandaSystem, Notify, StreamWriter, ConfigVariableString, ConfigPageManager
 from panda3d.direct import init_app_for_gui
@@ -570,14 +557,14 @@ class AppRunner(DirectObject):
             for packageData in hostData.packages:
                 totalSize += packageData.totalSize
         self.notify.info("Total Panda3D disk space used: %s MB" % (
-            (totalSize + 524288) / 1048576))
+            (totalSize + 524288) // 1048576))
 
         if self.verifyContents == self.P3DVCNever:
             # We're not allowed to delete anything anyway.
             return
 
         self.notify.info("Configured max usage is: %s MB" % (
-            (self.maxDiskUsage + 524288) / 1048576))
+            (self.maxDiskUsage + 524288) // 1048576))
         if totalSize <= self.maxDiskUsage:
             # Still within budget; no need to clean up anything.
             return
@@ -638,7 +625,7 @@ class AppRunner(DirectObject):
         except SystemExit as err:
             # Presumably the window has already been shut down here, but shut
             # it down again for good measure.
-            if hasattr(__builtin__, "base"):
+            if hasattr(builtins, "base"):
                 base.destroy()
 
             self.notify.info("Normal exit with status %s." % repr(err.code))
@@ -697,9 +684,10 @@ class AppRunner(DirectObject):
             # Replace the builtin open and file symbols so user code will get
             # our versions by default, which can open and read files out of
             # the multifile.
-            __builtin__.file = file.file
-            __builtin__.open = file.open
-            __builtin__.execfile = file.execfile
+            builtins.open = file.open
+            if sys.version_info < (3, 0):
+                builtins.file = file.open
+                builtins.execfile = file.execfile
             os.listdir = file.listdir
             os.walk = file.walk
             os.path.join = file.join

direct/src/p3d/HostInfo.py

@@ -161,14 +161,14 @@ class HostInfo:
                 self.notify.info("Downloading contents file %s" % (request))
                 statusCode = None
                 statusString = ''
-                for attempt in range(ConfigVariableInt('contents-xml-dl-attempts', 3)):
+                for attempt in range(int(ConfigVariableInt('contents-xml-dl-attempts', 3))):
                     if attempt > 0:
                         self.notify.info("Retrying (%s)..."%(attempt,))
                     rf = Ramfile()
                     channel = http.makeChannel(False)
                     channel.getDocument(request)
                     if channel.downloadToRam(rf):
-                        self.notify.warning("Successfully downloaded %s" % (url,))
+                        self.notify.info("Successfully downloaded %s" % (url,))
                         break
                     else:
                         rf = None
@@ -369,7 +369,7 @@ class HostInfo:
             assert self.hostDir
             self.__findHostXmlForHostDir(xcontents)
 
-        if not self.hostDir:
+        if self.rootDir and not self.hostDir:
             self.hostDir = self.__determineHostDir(None, self.hostUrl)
 
         # Get the list of packages available for download and/or import.
@@ -403,7 +403,7 @@ class HostInfo:
         self.hasContentsFile = True
 
         # Now save the contents.xml file into the standard location.
-        if not self.appRunner or self.appRunner.verifyContents != self.appRunner.P3DVCNever:
+        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCNever:
             assert self.hostDir
             filename = Filename(self.hostDir, 'contents.xml')
             filename.makeDir()
@@ -476,7 +476,7 @@ class HostInfo:
             self.descriptiveName = descriptiveName
 
         hostDirBasename = xhost.Attribute('host_dir')
-        if not self.hostDir:
+        if self.rootDir and not self.hostDir:
             self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl)
 
         # Get the "download" URL, which is the source from which we
@@ -514,8 +514,9 @@ class HostInfo:
 
         if not platform:
             # Ensure that we're on the same page with non-specified
-            # platforms.  We always use None, not empty string.
-            platform = None
+            # platforms.  We have to use the empty string, not None,
+            # since Python 3 can't sort lists with both strings and None.
+            platform = ""
 
         platforms = self.packages.setdefault((name, version), {})
         package = platforms.get(platform, None)
@@ -581,14 +582,12 @@ class HostInfo:
 
         result = []
 
-        items = self.packages.items()
-        items.sort()
+        items = sorted(self.packages.items())
         for key, platforms in items:
             if self.perPlatform or includeAllPlatforms:
                 # If we maintain a different answer per platform,
                 # return all of them.
-                pitems = platforms.items()
-                pitems.sort()
+                pitems = sorted(platforms.items())
                 for pkey, package in pitems:
                     result.append(package)
             else:
@@ -701,7 +700,7 @@ class HostInfo:
 
             # If we successfully got a hostname, we don't really need the
             # full hash.  We'll keep half of it.
-            keepHash = keepHash / 2;
+            keepHash = keepHash // 2
 
         md = HashVal()
         md.hashString(hostUrl)

direct/src/p3d/PackageInfo.py

@@ -277,7 +277,7 @@ class PackageInfo:
             # We've already got one.
             yield self.stepComplete; return
 
-        if self.host.appRunner and self.host.appRunner.verifyContents != self.host.appRunner.P3DVCNever:
+        if not self.host.appRunner or self.host.appRunner.verifyContents != self.host.appRunner.P3DVCNever:
             # We're allowed to download it.
             self.http = http
 
@@ -1151,7 +1151,7 @@ class PackageInfo:
         thisDir = ScanDirectoryNode(self.getPackageDir(), ignoreUsageXml = True)
         diskSpace = thisDir.getTotalSize()
         self.notify.info("Package %s uses %s MB" % (
-            self.packageName, (diskSpace + 524288) / 1048576))
+            self.packageName, (diskSpace + 524288) // 1048576))
         return diskSpace
 
     def markUsed(self):

direct/src/p3d/Packager.py

@@ -186,7 +186,7 @@ class Packager:
         def getKey(self):
             """ Returns a tuple used for sorting the PackageEntry
             objects uniquely per package. """
-            return (self.packageName, self.platform, self.version)
+            return (self.packageName, self.platform or "", self.version or "")
 
         def fromFile(self, packageName, platform, version, solo, perPlatform,
                      installDir, descFilename, importDescFilename):
@@ -300,8 +300,7 @@ class Packager:
                 xhost.InsertEndChild(xmirror)
 
             if packager:
-                altHosts = self.altHosts.items()
-                altHosts.sort()
+                altHosts = sorted(self.altHosts.items())
                 for keyword, alt in altHosts:
                     he = packager.hosts.get(alt, None)
                     if he:
@@ -393,8 +392,7 @@ class Packager:
                 raise PackagerError, message
 
             if self.ignoredDirFiles:
-                exts = list(self.ignoredDirFiles.keys())
-                exts.sort()
+                exts = sorted(self.ignoredDirFiles.keys())
                 total = sum([x for x in self.ignoredDirFiles.values()])
                 self.notify.warning("excluded %s files not marked for inclusion: %s" \
                                     % (total, ", ".join(["'" + ext + "'" for ext in exts])))
@@ -590,8 +588,7 @@ class Packager:
 
             # Add known module names.
             self.moduleNames = {}
-            modules = self.freezer.modules.items()
-            modules.sort()
+            modules = sorted(self.freezer.modules.items())
             for newName, mdef in modules:
                 if mdef.guess:
                     # Not really a module.
@@ -2766,11 +2763,15 @@ class Packager:
         # errors, and that the pdef file doesn't contain any really
         # crazy Python code, all this will do is fill in the
         # '__statements' list in the module scope.
+        fn = packageDef.toOsSpecific()
+        f = open(fn)
+        code = compile(f.read(), fn, 'exec')
+        f.close()
 
         # It appears that having a separate globals and locals
         # dictionary causes problems with resolving symbols within a
         # class scope.  So, we just use one dictionary, the globals.
-        execfile(packageDef.toOsSpecific(), globals)
+        exec(code, globals)
 
         packages = []
 
@@ -3351,42 +3352,27 @@ class Packager:
         producing their own custom panda3d for download.  Should be
         called before any other Python modules are named. """
 
-        # First, freeze just VFSImporter.py into its own
-        # _vfsimporter.pyd file.  This one is a special case, because
-        # we need this code in order to load python files from the
-        # Multifile, so this file can't itself be in the Multifile.
+        # This module and all its dependencies come frozen into p3dpython.
+        # We should mark them as having already been added so that we don't
+        # add them again to the Multifile.
+        self.do_module('direct.showbase.VFSImporter')
+        self.currentPackage.freezer.done(addStartupModules=True)
+        self.currentPackage.freezer.writeCode(None)
+        self.currentPackage.addExtensionModules()
+        self.currentPackage.freezer.reset()
 
-        # This requires a bit of care, because we only want to freeze
-        # VFSImporter.py, and not any other part of direct.  We do
-        # also want panda3d/__init__.py, though, since it would
-        # otherwise be part of the multifile.
-        self.do_excludeModule('direct')
-
-        # Import the actual VFSImporter module to get its filename on
-        # disk.
-        from direct.showbase import VFSImporter
-        filename = Filename.fromOsSpecific(VFSImporter.__file__)
-
-        self.do_module('VFSImporter', filename = filename)
-        self.do_freeze('_vfsimporter', compileToExe = False)
-
-        self.do_file('panda3d/_core.pyd');
-
-        # Now that we're done freezing, explicitly add 'direct' to
-        # counteract the previous explicit excludeModule().
-        self.do_module('direct')
+        self.do_file('panda3d/_core.pyd', newDir='panda3d')
 
         # This is the key Python module that is imported at runtime to
         # start an application running.
         self.do_module('direct.p3d.AppRunner')
 
         # This is the main program that drives the runtime Python.  It
-        # is responsible for loading _vfsimporter.pyd, and then
-        # importing direct.p3d.AppRunner, to start an application
-        # running.  The program comes in two parts: an executable, and
-        # an associated dynamic library.  Note that the .exe and .dll
-        # extensions are automatically replaced with the appropriate
-        # platform-specific extensions.
+        # is responsible for importing direct.p3d.AppRunner to start an
+        # application running.  The program comes in two parts: an
+        # executable, and an associated dynamic library.  Note that the
+        # .exe and .dll extensions are automatically replaced with the
+        # appropriate platform-specific extensions.
 
         if self.platform.startswith('osx'):
             # On Mac, we package up a P3DPython.app bundle.  This
@@ -3469,7 +3455,7 @@ class Packager:
                 freezer.addModule(moduleName, newName = newName)
             else:
                 freezer.modules[newName] = freezer.modules[moduleName]
-        freezer.done(compileToExe = compileToExe)
+        freezer.done(addStartupModules = compileToExe)
 
         dirname = ''
         basename = filename
@@ -3850,8 +3836,7 @@ class Packager:
                 xhost = he.makeXml(packager = self)
                 xcontents.InsertEndChild(xhost)
 
-        contents = self.contents.items()
-        contents.sort()
+        contents = sorted(self.contents.items())
         for key, pe in contents:
             xpackage = pe.makeXml()
             xcontents.InsertEndChild(xpackage)

direct/src/p3d/packp3d.py

@@ -164,10 +164,10 @@ def makePackedApp(args):
                 PandaSystem.getPackageHostUrl(),
                 os.path.split(sys.argv[0])[1],
                 '%s.%s' % (sys.version_info[0], sys.version_info[1]))
-            sys.exit(1)
+            sys.exit(0)
 
     if not appFilename:
-        raise ArgumentError, "No target app specified.  Use:\n%s -o app.p3d" % (os.path.split(sys.argv[0])[1])
+        raise ArgumentError, "No target app specified.  Use:\n  %s -o app.p3d\nUse -h to get more usage information." % (os.path.split(sys.argv[0])[1])
 
     if args:
         raise ArgumentError, "Extra arguments on command line."

direct/src/p3d/panda3d.pdef

@@ -61,7 +61,7 @@ class panda3d(package):
            'panda3d.physics')
 
     # Include various standard Python encodings.  The rest is in morepy.
-    module('encodings', 'encodings.aliases', 'encodings.undefined,'
+    module('encodings', 'encodings.aliases', 'encodings.undefined',
            'encodings.utf_8', 'encodings.ascii', 'encodings.string_escape',
            'encodings.mbcs', 'encodings.latin_1', 'io')
 

direct/src/plugin/p3dPythonRun.cxx

@@ -21,6 +21,9 @@
 
 #include "py_panda.h"
 
+// This has been compiled-in by the build system, if all is well.
+extern struct _frozen _PyImport_FrozenModules[];
+
 // There is only one P3DPythonRun object in any given process space.
 // Makes the statics easier to deal with, and we don't need multiple
 // instances of this thing.
@@ -85,17 +88,30 @@ P3DPythonRun(const char *program_name, const char *archive_file,
   // Turn off the automatic load of site.py at startup.
   extern int Py_NoSiteFlag;
   Py_NoSiteFlag = 1;
+  Py_NoUserSiteDirectory = 1;
+
+  // Tell Python not to write bytecode files for loaded modules.
+  Py_DontWriteBytecodeFlag = 1;
+
+  // Prevent Python from complaining about finding the standard modules.
+  Py_FrozenFlag = 1;
+
+  // This contains the modules we need in order to call Py_Initialize,
+  // as well as the VFSImporter.
+  PyImport_FrozenModules = _PyImport_FrozenModules;
 
   // Initialize Python.  It appears to be important to do this before
   // we open the pipe streams and spawn the thread, below.
 #if PY_MAJOR_VERSION >= 3
   Py_SetProgramName((wchar_t *)_program_name.c_str());
+  Py_SetPythonHome((wchar_t *)L"");
 #else
   Py_SetProgramName((char *)_program_name.c_str());
+  Py_SetPythonHome((char *)"");
 #endif
   Py_Initialize();
   PyEval_InitThreads();
-  PySys_SetArgv(_py_argc, _py_argv);
+  PySys_SetArgvEx(_py_argc, _py_argv, 0);
 
   // Open the error output before we do too much more.
   if (log_pathname != NULL && *log_pathname != '\0') {
@@ -170,38 +186,44 @@ run_python() {
   // setting __path__ of frozen modules properly.
   PyObject *panda3d_module = PyImport_AddModule("panda3d");
   if (panda3d_module == NULL) {
-    nout << "Failed to create panda3d module:\n";
+    nout << "Failed to add panda3d module:\n";
     PyErr_Print();
     return 1;
   }
 
-  // Set the __path__ such that it can find panda3d/core.pyd, etc.
+  // Set the __path__ such that it can find panda3d/_core.pyd, etc.
   Filename panda3d_dir(dir, "panda3d");
   string dir_str = panda3d_dir.to_os_specific();
-  PyObject *panda3d_dict = PyModule_GetDict(panda3d_module);
-  PyObject *panda3d_path = Py_BuildValue("[s#]", dir_str.data(), dir_str.length());
-  PyDict_SetItemString(panda3d_dict, "__path__", panda3d_path);
-  Py_DECREF(panda3d_path);
-
-  // Now we can load _vfsimporter.pyd.  Since this is a magic frozen
-  // pyd, importing it automatically makes all of its frozen contents
-  // available to import as well.
-  PyObject *vfsimporter = PyImport_ImportModule("_vfsimporter");
-  if (vfsimporter == NULL) {
-    nout << "Failed to import _vfsimporter:\n";
-    PyErr_Print();
-    return 1;
-  }
-  Py_DECREF(vfsimporter);
+  PyModule_AddObject(panda3d_module, "__path__", Py_BuildValue("[s#]", dir_str.data(), dir_str.length()));
+  PyModule_AddStringConstant(panda3d_module, "__package__", "panda3d");
 
-  // And now we can import the VFSImporter module that was so defined.
-  PyObject *vfsimporter_module = PyImport_ImportModule("VFSImporter");
+  // Import the VFSImporter module that was frozen in.
+  PyObject *vfsimporter_module = PyImport_ImportModule("direct.showbase.VFSImporter");
   if (vfsimporter_module == NULL) {
     nout << "Failed to import VFSImporter:\n";
     PyErr_Print();
     return 1;
   }
 
+  // Now repair the "direct" and "direct.showbase" trees, which were
+  // presumably frozen along with the VFSImporter, by setting their
+  // __path__ such that we can still find the other direct modules.
+  Filename direct_dir(dir, "direct");
+  PyObject *direct_module = PyImport_AddModule("direct");
+  if (direct_module != NULL) {
+    dir_str = direct_dir.to_os_specific();
+    PyModule_AddObject(direct_module, "__path__", Py_BuildValue("[s#]", dir_str.data(), dir_str.length()));
+    PyModule_AddStringConstant(direct_module, "__package__", "direct");
+  }
+
+  PyObject *showbase_module = PyImport_AddModule("direct.showbase");
+  if (showbase_module != NULL) {
+    Filename showbase_dir(direct_dir, "showbase");
+    dir_str = showbase_dir.to_os_specific();
+    PyModule_AddObject(showbase_module, "__path__", Py_BuildValue("[s#]", dir_str.data(), dir_str.length()));
+    PyModule_AddStringConstant(showbase_module, "__package__", "direct.showbase");
+  }
+
   // And register the VFSImporter.
   PyObject *result = PyObject_CallMethod(vfsimporter_module, (char *)"register", (char *)"");
   if (result == NULL) {

direct/src/showbase/VFSImporter.py

@@ -136,7 +136,7 @@ class VFSLoader:
 
         code = self._read_code()
         if not code:
-            raise ImportError, 'No Python code in %s' % (fullname)
+            raise ImportError('No Python code in %s' % (fullname))
 
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = self.filename.toOsSpecific()
@@ -230,6 +230,10 @@ class VFSLoader:
         #print >>sys.stderr, "importing frozen %s" % (fullname)
         module = imp.load_module(fullname, None, fullname,
                                  ('', '', imp.PY_FROZEN))
+
+        # Workaround for bug in Python 2.
+        if getattr(module, '__path__', None) == fullname:
+            module.__path__ = []
         return module
 
     def _read_code(self):
@@ -243,7 +247,7 @@ class VFSLoader:
             pycVfile = vfs.getFile(self.filename, False)
             if pycVfile:
                 return self._loadPyc(pycVfile, None)
-            raise IOError, 'Could not read %s' % (self.filename)
+            raise IOError('Could not read %s' % (self.filename))
 
         elif self.fileType == FTExtensionModule:
             return None
@@ -281,16 +285,21 @@ class VFSLoader:
 
         code = None
         data = vfile.readFile(True)
-        if data[:4] == imp.get_magic():
+        if data[:4] != imp.get_magic():
+            raise ValueError("Bad magic number in %s" % (vfile))
+
+        if sys.version_info >= (3, 0):
+            t = int.from_bytes(data[4:8], 'little')
+            data = data[12:]
+        else:
             t = ord(data[4]) + (ord(data[5]) << 8) + \
                (ord(data[6]) << 16) + (ord(data[7]) << 24)
-            if not timestamp or t == timestamp:
-                code = marshal.loads(data[8:])
-            else:
-                raise ValueError, 'Timestamp wrong on %s' % (vfile)
+            data = data[8:]
+
+        if not timestamp or t == timestamp:
+            return marshal.loads(data)
         else:
-            raise ValueError, 'Bad magic number in %s' % (vfile)
-        return code
+            raise ValueError("Timestamp wrong on %s" % (vfile))
 
 
     def _compile(self, filename, source):
@@ -310,15 +319,16 @@ class VFSLoader:
         except IOError:
             pass
         else:
-            f.write('\0\0\0\0')
-            f.write(chr(self.timestamp & 0xff) +
-                    chr((self.timestamp >> 8) & 0xff) +
-                    chr((self.timestamp >> 16) & 0xff) +
-                    chr((self.timestamp >> 24) & 0xff))
-            f.write(marshal.dumps(code))
-            f.flush()
-            f.seek(0, 0)
             f.write(imp.get_magic())
+            if sys.version_info >= (3, 0):
+                f.write((self.timestamp & 0xffffffff).to_bytes(4, 'little'))
+                f.write(b'\0\0\0\0')
+            else:
+                f.write(chr(self.timestamp & 0xff) +
+                        chr((self.timestamp >> 8) & 0xff) +
+                        chr((self.timestamp >> 16) & 0xff) +
+                        chr((self.timestamp >> 24) & 0xff))
+            f.write(marshal.dumps(code))
             f.close()
 
         return code
@@ -388,7 +398,7 @@ class VFSSharedImporter:
         """ Returns the directory name that the indicated
         conventionally-loaded module must have been loaded from. """
 
-        if not hasattr(mod, __file__) or mod.__file__ is None:
+        if not getattr(mod, '__file__', None):
             return None
 
         fullname = mod.__name__
@@ -433,6 +443,9 @@ class VFSSharedLoader:
         if self.reload:
             mod = sys.modules[fullname]
             path = mod.__path__ or []
+            if path == fullname:
+                # Work around Python bug setting __path__ of frozen modules.
+                path = []
             vfs_shared_path = getattr(mod, '_vfs_shared_path', [])
 
         for loader in self.loaders:
@@ -450,11 +463,12 @@ class VFSSharedLoader:
 
         if mod is None:
             # If all of them failed to load, raise ImportError.
-            raise ImportError, message
+            raise ImportError(message)
 
         # If at least one of them loaded successfully, return the
         # union of loaded modules.
         mod.__path__ = path
+        mod.__package__ = fullname
 
         # Also set this special symbol, which records that this is a
         # shared package, and also lists the paths we have already
@@ -515,7 +529,9 @@ def reloadSharedPackages():
 
     #print >> sys.stderr, "reloadSharedPackages, path = %s, sharedPackages = %s" % (sys.path, sharedPackages.keys())
 
-    for fullname in sharedPackages.keys():
+    # Sort the list, just to make sure parent packages are reloaded
+    # before child packages are.
+    for fullname in sorted(sharedPackages.keys()):
         mod = sys.modules.get(fullname, None)
         if not mod:
             continue

direct/src/showutil/FreezeTool.py

@@ -31,14 +31,12 @@ isDebugBuild = (python.lower().endswith('_d'))
 # must be frozen in any main.exe.
 startupModules = [
     'site', 'sitecustomize', 'os', 'encodings.cp1252',
-    'org',
+    'encodings.latin_1', 'encodings.utf_8', 'io', 'org',
     ]
 
 # These are missing modules that we've reported already this session.
 reportedMissing = {}
 
-# Our own Python source trees to watch out for.
-sourceTrees = ['direct']
 
 class CompilationEnvironment:
     """ Create an instance of this class to record the commands to
@@ -445,7 +443,7 @@ extend_frozen_modules(const struct _frozen *new_modules, int new_count) {
 }
 
 %(dllexport)svoid init%(moduleName)s() {
-  extend_frozen_modules(_PyImport_FrozenModules, %(newcount)s);
+  extend_frozen_modules(_PyImport_FrozenModules, sizeof(_PyImport_FrozenModules) / sizeof(struct _frozen));
   Py_InitModule("%(moduleName)s", nullMethods);
 }
 """
@@ -458,7 +456,7 @@ programFile = """
 
 %(moduleDefs)s
 
-static struct _frozen _PyImport_FrozenModules[] = {
+struct _frozen _PyImport_FrozenModules[] = {
 %(moduleList)s
   {NULL, NULL, 0}
 };
@@ -485,7 +483,7 @@ okMissing = [
     'Carbon.Folder', 'Carbon.Folders', 'HouseGlobals', 'Carbon.File',
     'MacOS', '_emx_link', 'ce', 'mac', 'org.python.core', 'os.path',
     'os2', 'posix', 'pwd', 'readline', 'riscos', 'riscosenviron',
-    'riscospath', 'dbm', 'fcntl', 'win32api',
+    'riscospath', 'dbm', 'fcntl', 'win32api', 'usercustomize',
     '_winreg', 'ctypes', 'ctypes.wintypes', 'nt','msvcrt',
     'EasyDialogs', 'SOCKS', 'ic', 'rourl2path', 'termios',
     'OverrideFrom23._Res', 'email', 'email.Utils', 'email.Generator',
@@ -613,20 +611,14 @@ class Freezer:
 
         self.mf = None
 
-        # Make sure we know how to find "direct".
-        for sourceTree in sourceTrees:
-            try:
-                module = __import__(sourceTree)
-            except:
-                pass
-
         # Actually, make sure we know how to find all of the
         # already-imported modules.  (Some of them might do their own
         # special path mangling.)
         for moduleName, module in sys.modules.items():
             if module and hasattr(module, '__path__'):
                 path = getattr(module, '__path__')
-                modulefinder.AddPackagePath(moduleName, path[0])
+                if path:
+                    modulefinder.AddPackagePath(moduleName, path[0])
 
     def excludeFrom(self, freezer):
         """ Excludes all modules that have already been processed by
@@ -827,7 +819,7 @@ class Freezer:
                 moduleName, filename = filename, implicit = implicit,
                 guess = guess, fromSource = fromSource, text = text)
 
-    def done(self, compileToExe = False):
+    def done(self, addStartupModules = False):
         """ Call this method after you have added all modules with
         addModule().  You may then call generateCode() or
         writeMultifile() to dump the resulting output.  After a call
@@ -838,7 +830,9 @@ class Freezer:
 
         # If we are building an exe, we also need to implicitly
         # bring in Python's startup modules.
-        if compileToExe:
+        if addStartupModules:
+            self.modules['_frozen_importlib'] = self.ModuleDef('importlib._bootstrap', implicit = True)
+
             for moduleName in startupModules:
                 if moduleName not in self.modules:
                     self.modules[moduleName] = self.ModuleDef(moduleName, implicit = True)
@@ -1071,8 +1065,12 @@ class Freezer:
 
     def __addPyc(self, multifile, filename, code, compressionLevel):
         if code:
-            data = imp.get_magic() + b'\0\0\0\0' + \
-                   marshal.dumps(code)
+            data = imp.get_magic() + b'\0\0\0\0'
+
+            if sys.version_info >= (3, 0):
+                data += b'\0\0\0\0'
+
+            data += marshal.dumps(code)
 
             stream = StringStream(data)
             multifile.addSubfile(filename, stream, compressionLevel)
@@ -1214,22 +1212,9 @@ class Freezer:
         multifile.flush()
         multifile.repack()
 
-    def generateCode(self, basename, compileToExe = False):
-        """ After a call to done(), this freezes all of the
-        accumulated python code into either an executable program (if
-        compileToExe is true) or a dynamic library (if compileToExe is
-        false).  The basename is the name of the file to write,
-        without the extension.
-
-        The return value is the newly-generated filename, including
-        the filename extension.  Additional extension modules are
-        listed in self.extras. """
-
-        if compileToExe:
-            # We must have a __main__ module to make an exe file.
-            if not self.__writingModule('__main__'):
-                message = "Can't generate an executable without a __main__ module."
-                raise StandardError, message
+    def writeCode(self, filename, initCode = ""):
+        """ After a call to done(), this freezes all of the accumulated
+        Python code into a C source file. """
 
         self.__replacePaths()
 
@@ -1247,38 +1232,57 @@ class Freezer:
                 # Allow importing this module.
                 module = self.mf.modules.get(origName, None)
                 code = getattr(module, "__code__", None)
-                if not code and moduleName in startupModules:
+                if code:
+                    code = marshal.dumps(code)
+
+                    mangledName = self.mangleName(moduleName)
+                    moduleDefs.append(self.makeModuleDef(mangledName, code))
+                    moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, module))
+
+                elif moduleName in startupModules:
                     # Forbid the loading of this startup module.
                     moduleList.append(self.makeForbiddenModuleListEntry(moduleName))
+
                 else:
-                    if origName in sourceTrees:
-                        # This is one of Panda3D's own Python source
-                        # trees.  These are a special case: we don't
-                        # compile the __init__.py files within them,
-                        # since their only purpose is to munge the
-                        # __path__ variable anyway.  Instead, we
-                        # pretend the __init__.py files are empty.
-                        code = compile('', moduleName, 'exec')
-
-                    if code:
-                        code = marshal.dumps(code)
-
-                        mangledName = self.mangleName(moduleName)
-                        moduleDefs.append(self.makeModuleDef(mangledName, code))
-                        moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, module))
+                    # This is a module with no associated Python
+                    # code.  It must be an extension module.  Get the
+                    # filename.
+                    extensionFilename = getattr(module, '__file__', None)
+                    if extensionFilename:
+                        self.extras.append((moduleName, extensionFilename))
                     else:
+                        # It doesn't even have a filename; it must
+                        # be a built-in module.  No worries about
+                        # this one, then.
+                        pass
 
-                        # This is a module with no associated Python
-                        # code.  It must be an extension module.  Get the
-                        # filename.
-                        extensionFilename = getattr(module, '__file__', None)
-                        if extensionFilename:
-                            self.extras.append((moduleName, extensionFilename))
-                        else:
-                            # It doesn't even have a filename; it must
-                            # be a built-in module.  No worries about
-                            # this one, then.
-                            pass
+        text = programFile % {
+            'moduleDefs': '\n'.join(moduleDefs),
+            'moduleList': '\n'.join(moduleList),
+            'initCode': initCode
+            }
+
+        if filename is not None:
+            file = open(filename, 'w')
+            file.write(text)
+            file.close()
+
+    def generateCode(self, basename, compileToExe = False):
+        """ After a call to done(), this freezes all of the
+        accumulated python code into either an executable program (if
+        compileToExe is true) or a dynamic library (if compileToExe is
+        false).  The basename is the name of the file to write,
+        without the extension.
+
+        The return value is the newly-generated filename, including
+        the filename extension.  Additional extension modules are
+        listed in self.extras. """
+
+        if compileToExe:
+            # We must have a __main__ module to make an exe file.
+            if not self.__writingModule('__main__'):
+                message = "Can't generate an executable without a __main__ module."
+                raise StandardError, message
 
         filename = basename + self.sourceExtension
 
@@ -1317,21 +1321,12 @@ class Freezer:
 
             initCode = dllInitCode % {
                 'moduleName' : os.path.basename(basename),
-                'newcount' : len(moduleList),
                 'dllexport' : dllexport,
                 'dllimport' : dllimport,
                 }
             compileFunc = self.cenv.compileDll
 
-        text = programFile % {
-            'moduleDefs' : '\n'.join(moduleDefs),
-            'moduleList' : '\n'.join(moduleList),
-            'initCode' : initCode,
-            }
-
-        file = open(filename, 'w')
-        file.write(text)
-        file.close()
+        self.writeCode(filename, initCode=initCode)
 
         try:
             compileFunc(filename, basename)
@@ -1390,9 +1385,9 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
     def __init__(self, *args, **kw):
         modulefinder.ModuleFinder.__init__(self, *args, **kw)
 
-    def find_module(self, name, path, parent=None):
+    def find_module(self, name, path, *args, **kwargs):
         try:
-            return modulefinder.ModuleFinder.find_module(self, name, path, parent = parent)
+            return modulefinder.ModuleFinder.find_module(self, name, path, *args, **kwargs)
         except ImportError:
             # It wasn't found through the normal channels.  Maybe it's
             # one of ours, or maybe it's frozen?

direct/src/showutil/pfreeze.py

@@ -40,6 +40,11 @@ Options:
      of the __path__ variable, and thus must be actually imported to
      determine the true value of __path__.
 
+  -s
+     Adds the standard set of modules that are necessary for embedding
+     the Python interpreter.  Implicitly set if an executable is
+     generated.
+
 """
 
 import getopt
@@ -58,9 +63,10 @@ def usage(code, msg = ''):
 freezer = FreezeTool.Freezer()
 
 basename = None
+addStartupModules = False
 
 try:
-    opts, args = getopt.getopt(sys.argv[1:], 'o:i:x:p:h')
+    opts, args = getopt.getopt(sys.argv[1:], 'o:i:x:p:sh')
 except getopt.error, msg:
     usage(1, msg)
 
@@ -76,48 +82,57 @@ for opt, arg in opts:
     elif opt == '-p':
         for module in arg.split(','):
             freezer.handleCustomPath(module)
+    elif opt == '-s':
+        addStartupModules = True
     elif opt == '-h':
         usage(0)
     else:
         print 'illegal option: ' + flag
         sys.exit(1)
 
-if not args:
-    usage(0)
-
 if not basename:
     usage(1, 'You did not specify an output file.')
 
-if len(args) != 1:
+if len(args) > 1:
     usage(1, 'Only one main file may be specified.')
 
 outputType = 'exe'
 bl = basename.lower()
 if bl.endswith('.mf'):
     outputType = 'mf'
+elif bl.endswith('.c'):
+    outputType = 'c'
 elif bl.endswith('.dll') or bl.endswith('.pyd') or bl.endswith('.so'):
     basename = os.path.splitext(basename)[0]
     outputType = 'dll'
 elif bl.endswith('.exe'):
     basename = os.path.splitext(basename)[0]
 
-startfile = args[0]
-startmod = startfile
-if startfile.endswith('.py') or startfile.endswith('.pyw') or \
-   startfile.endswith('.pyc') or startfile.endswith('.pyo'):
-    startmod = os.path.splitext(startfile)[0]
-
 compileToExe = False
-if outputType == 'dll':
-    freezer.addModule(startmod, filename = startfile)
-else:
-    freezer.addModule('__main__', filename = startfile)
-    compileToExe = True
+if args:
+    startfile = args[0]
+    startmod = startfile
+    if startfile.endswith('.py') or startfile.endswith('.pyw') or \
+       startfile.endswith('.pyc') or startfile.endswith('.pyo'):
+        startmod = os.path.splitext(startfile)[0]
+
+    if outputType == 'dll' or outputType == 'c':
+        freezer.addModule(startmod, filename = startfile)
+    else:
+        freezer.addModule('__main__', filename = startfile)
+        compileToExe = True
+        addStartupModules = True
+
+elif outputType == 'exe':
+    # We must have a main module when making an executable.
+    usage(0)
 
-freezer.done(compileToExe = compileToExe)
+freezer.done(addStartupModules = addStartupModules)
 
 if outputType == 'mf':
     freezer.writeMultifile(basename)
+elif outputType == 'c':
+    freezer.writeCode(basename)
 else:
     freezer.generateCode(basename, compileToExe = compileToExe)
 

direct/src/stdpy/file.py

@@ -5,175 +5,185 @@ SIMPLE_THREADS model, by avoiding blocking all threads while waiting
 for I/O to complete. """
 
 __all__ = [
-    'file', 'open', 'listdir', 'walk', 'join',
+    'open', 'listdir', 'walk', 'join',
     'isfile', 'isdir', 'exists', 'lexists', 'getmtime', 'getsize',
     'execfile',
     ]
 
-from panda3d import core
+import panda3d._core as core
 import sys
-import types
 import os
+import io
 
 _vfs = core.VirtualFileSystem.getGlobalPtr()
 
-class file:
-    def __init__(self, filename, mode = 'r', bufsize = None,
-                 autoUnwrap = False):
-        self.__stream = None
-        self.__needsVfsClose = False
-        self.__reader = None
-        self.__writer = None
-        self.closed = True
-        self.encoding = None
-        self.errors = None
-        self.__lastWrite = False
-
-        self.mode = mode
-        self.name = None
-        self.filename = None
-        self.newlines = None
-        self.softspace = False
-
-        readMode = False
-        writeMode = False
-
-        if isinstance(filename, core.Istream) or isinstance(filename, core.Ostream):
-            # If we were given a stream instead of a filename, assign
-            # it directly.
-            self.__stream = filename
-            readMode = isinstance(filename, core.Istream)
-            writeMode = isinstance(filename, core.Ostream)
-
-        elif isinstance(filename, core.VirtualFile):
+def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
+    if sys.version_info >= (3, 0):
+        # Python 3 is much stricter than Python 2, which lets
+        # unknown flags fall through.
+        for ch in mode:
+            if ch not in 'rwxabt+U':
+                raise IOError("invalid mode: '%s'" % (mode))
+
+    creating = 'x' in mode
+    writing = 'w' in mode
+    appending = 'a' in mode
+    updating = '+' in mode
+    binary = 'b' in mode
+    universal = 'U' in mode
+    reading = universal or 'r' in mode
+
+    if binary and 't' in mode:
+        raise IOError("can't have text and binary mode at once")
+
+    if creating + reading + writing + appending > 1:
+        raise ValueError("must have exactly one of create/read/write/append mode")
+
+    if binary:
+        if encoding:
+            raise ValueError("binary mode doesn't take an encoding argument")
+        if errors:
+            raise ValueError("binary mode doesn't take an errors argument")
+        if newline:
+            raise ValueError("binary mode doesn't take a newline argument")
+
+    if isinstance(file, core.Istream) or isinstance(file, core.Ostream):
+        # If we were given a stream instead of a filename, assign
+        # it directly.
+        raw = StreamIOWrapper(file)
+        raw.mode = mode
+
+    else:
+        vfile = None
+
+        if isinstance(file, core.VirtualFile):
             # We can also "open" a VirtualFile object for reading.
-            self.__stream = filename.openReadFile(autoUnwrap)
-            if not self.__stream:
-                message = 'Could not read virtual file %s' % (repr(filename))
-                raise IOError, message
-            self.__needsVfsClose = True
-            readMode = True
-
+            vfile = file
+            filename = vfile.getFilename()
+            filename.setBinary()
         else:
             # Otherwise, we must have been given a filename.  Open it.
-            if isinstance(filename, types.StringTypes):
+            if isinstance(file, unicode):
                 # If a raw string is given, assume it's an os-specific
                 # filename.
-                filename = core.Filename.fromOsSpecific(filename)
+                filename = core.Filename.fromOsSpecificW(file)
+            elif isinstance(file, str):
+                filename = core.Filename.fromOsSpecific(file)
             else:
                 # If a Filename is given, make a writable copy anyway.
-                filename = core.Filename(filename)
-
-            self.filename = filename
-            self.name = filename.toOsSpecific()
-
-            if sys.version_info >= (3, 0):
-                # Python 3 is much stricter than Python 2, which lets
-                # unknown flags fall through.
-                for ch in mode:
-                    if not ch in 'rwxabt+U':
-                        raise IOError("invalid mode: " + mode)
-
-            binary = False
-            if 'b' in mode and 't' in mode:
-                raise IOError("can't have text and binary mode at once")
-            
-            if 'b' in mode:
-                # Strip 'b'.  This means a binary file.
-                i = mode.index('b')
-                mode = mode[:i] + mode[i + 1:]
-                binary = True
-            elif 't' in mode:
-                # Strip 't'.  This means a text file (redundant, yes).
-                i = mode.index('t')
-                mode = mode[:i] + mode[i + 1:]
-                binary = False
-
-            if 'U' in mode:
-                # Strip 'U'.  We don't use it; universal-newline support
-                # is built into Panda, and can't be changed at runtime.
-                i = mode.index('U')
-                mode = mode[:i] + mode[i + 1:]
-                binary = False
-
-            if mode == '':
-                mode = 'r'
-
-            # Per Python docs, we insist this is true.
-            modeType = mode[0]
-            assert modeType in 'rwa'
-
-            if binary:
-                filename.setBinary()
+                filename = core.Filename(file)
+
+            filename.setBinary()
+            vfile = _vfs.getFile(filename)
+
+        if not vfile:
+            if reading:
+                raise FileNotFoundError("No such file or directory: '%s'" % (filename))
+
+            vfile = _vfs.createFile(filename)
+            if not vfile:
+                raise IOError("Failed to create file: '%s'" % (filename))
+
+        elif creating:
+            # In 'creating' mode, we have to raise FileExistsError
+            # if the file already exists.  Otherwise, it's the same
+            # as 'writing' mode.
+            raise FileExistsError("File exists: '%s'" % (filename))
+
+        elif vfile.isDirectory():
+            raise IsADirectoryError("Is a directory: '%s'" % (filename))
+
+        # Actually open the streams.
+        if reading:
+            if updating:
+                stream = vfile.openReadWriteFile(False)
             else:
-                filename.setText()
-            
-            # Actually open the streams, taking care to 
-            # ignore unknown chars in the mode string.
-            # We already asserted that it starts with a mode
-            # char above, so locate the '+'  
-            if modeType == 'w' and '+' in mode:
-                self.__stream = _vfs.openReadWriteFile(filename, True)
-                if not self.__stream:
-                    message = 'Could not open %s for writing' % (filename)
-                    raise IOError(message)
-                readMode = True
-                writeMode = True
-
-            elif modeType == 'a' and '+' in mode:
-                self.__stream = _vfs.openReadAppendFile(filename)
-                if not self.__stream:
-                    message = 'Could not open %s for writing' % (filename)
-                    raise IOError(message)
-                readMode = True
-                writeMode = True
-
-            elif modeType == 'r' and '+' in mode:
-                self.__stream = _vfs.openReadWriteFile(filename, False)
-                if not self.__stream:
-                    message = 'Could not open %s for writing' % (filename)
-                    raise IOError(message)
-                readMode = True
-                writeMode = True
-                
-            elif modeType == 'w':
-                self.__stream = _vfs.openWriteFile(filename, autoUnwrap, True)
-                if not self.__stream:
-                    message = 'Could not open %s for writing' % (filename)
-                    raise IOError(message)
-                writeMode = True
-
-            elif modeType == 'a':
-                self.__stream = _vfs.openAppendFile(filename)
-                if not self.__stream:
-                    message = 'Could not open %s for writing' % (filename)
-                    raise IOError(message)
-                writeMode = True
-
-            elif modeType == 'r':
-                self.__stream = _vfs.openReadFile(filename, autoUnwrap)
-                if not self.__stream:
-                    if not _vfs.exists(filename):
-                        message = 'No such file: %s' % (filename)
-                    else:
-                        message = 'Could not open %s for reading' % (filename)
-                    raise IOError(message)
-                readMode = True
-                
+                stream = vfile.openReadFile(False)
+
+            if not stream:
+                raise IOError("Could not open %s for reading" % (filename))
+
+        elif writing or creating:
+            if updating:
+                stream = vfile.openReadWriteFile(True)
+            else:
+                stream = vfile.openWriteFile(False)
+
+            if not stream:
+                raise IOError("Could not open %s for writing" % (filename))
+
+        elif appending:
+            if updating:
+                stream = vfile.openReadAppendFile()
             else:
-                # should not get here unless there's a bug above
-                raise IOError("Unhandled mode flags: " + mode)
+                stream = vfile.openAppendFile()
+
+            if not stream:
+                raise IOError("Could not open %s for appending" % (filename))
+
+        else:
+            raise ValueError("Must have exactly one of create/read/write/append mode and at most one plus")
+
+        raw = StreamIOWrapper(stream, needsVfsClose=True)
+        raw.mode = mode
+        raw.name = vfile.getFilename().toOsSpecific()
+
+    # If a binary stream was requested, return the stream we've created.
+    if binary:
+        return raw
+
+    line_buffering = False
+    if buffering == 1:
+        line_buffering = True
+    elif buffering == 0:
+        raise ValueError("can't have unbuffered text I/O")
 
-            self.__needsVfsClose = True
+    # Otherwise, create a TextIOWrapper object to wrap it.
+    wrapper = io.TextIOWrapper(raw, encoding, errors, newline, line_buffering)
+    wrapper.mode = mode
+    return wrapper
 
-        if readMode:
+
+if sys.version_info < (3, 0):
+    # Python 2 had an alias for open() called file().
+    __all__.append('file')
+    file = open
+
+
+class StreamIOWrapper(io.IOBase):
+    """ This is a file-like object that wraps around a C++ istream and/or
+    ostream object.  It only deals with binary data; to work with text I/O,
+    create an io.TextIOWrapper object around this, or use the open()
+    function that is also provided with this module. """
+
+    def __init__(self, stream, needsVfsClose=False):
+        self.__stream = stream
+        self.__needsVfsClose = needsVfsClose
+        self.__reader = None
+        self.__writer = None
+        self.__lastWrite = False
+
+        if isinstance(stream, core.Istream):
             self.__reader = core.StreamReader(self.__stream, False)
-        if writeMode:
-            self.__writer = core.StreamWriter(self.__stream, False)
+
+        if isinstance(stream, core.Ostream):
+            self.__writer = core.StreamWriter(stream, False)
             self.__lastWrite = True
 
-    def __del__(self):
-        self.close()
+    def __repr__(self):
+        s = "<direct.stdpy.file.StreamIOWrapper"
+        if hasattr(self, 'name'):
+            s += " name='%s'" % (self.name)
+        if hasattr(self, 'mode'):
+            s += " mode='%s'" % (self.mode)
+        s += ">"
+        return s
+
+    def readable(self):
+        return self.__reader is not None
+
+    def writable(self):
+        return self.__writer is not None
 
     def close(self):
         if self.__needsVfsClose:
@@ -185,70 +195,49 @@ class file:
                 _vfs.closeWriteFile(self.__stream)
 
             self.__needsVfsClose = False
+
         self.__stream = None
-        self.__needsVfsClose = False
         self.__reader = None
         self.__writer = None
 
     def flush(self):
-        if self.__stream:
+        if self.__writer:
             self.__stream.clear()  # clear eof flag
             self.__stream.flush()
 
-    def __iter__(self):
-        return self
-
-    def next(self):
-        line = self.readline()
-        if line:
-            return line
-        raise StopIteration
-
-    def read(self, size = -1):
+    def read(self, size=-1):
         if not self.__reader:
             if not self.__writer:
                 # The stream is not even open at all.
-                message = 'I/O operation on closed file'
-                raise ValueError, message
+                raise ValueError("I/O operation on closed file")
+
             # The stream is open only in write mode.
-            message = 'Attempt to read from write-only stream'
-            raise IOError, message
+            raise IOError("Attempt to read from write-only stream")
 
         self.__stream.clear()  # clear eof flag
         self.__lastWrite = False
-        if size >= 0:
+        if size is not None and size >= 0:
             result = self.__reader.extractBytes(size)
         else:
             # Read to end-of-file.
-            result = ''
+            result = b''
             while not self.__stream.eof():
-                result += self.__reader.extractBytes(1024)
+                result += self.__reader.extractBytes(512)
         return result
 
-    def readline(self, size = -1):
+    def readline(self, size=-1):
         if not self.__reader:
             if not self.__writer:
                 # The stream is not even open at all.
-                message = 'I/O operation on closed file'
-                raise ValueError, message
+                raise ValueError("I/O operation on closed file")
+
             # The stream is open only in write mode.
-            message = 'Attempt to read from write-only stream'
-            raise IOError, message
+            raise IOError("Attempt to read from write-only stream")
 
         self.__stream.clear()  # clear eof flag
         self.__lastWrite = False
         return self.__reader.readline()
 
-    def readlines(self, sizehint = -1):
-        lines = []
-        line = self.readline()
-        while line:
-            lines.append(line)
-            line = self.readline()
-        return lines
-
-    xreadlines = readlines
-
     def seek(self, offset, whence = 0):
         if self.__stream:
             self.__stream.clear()  # clear eof flag
@@ -264,58 +253,43 @@ class file:
         else:
             if self.__reader:
                 return self.__stream.tellg()
-        message = 'I/O operation on closed file'
-        raise ValueError, message
+        raise ValueError("I/O operation on closed file")
 
-    def truncate(self):
-        """ Sorry, this isn't supported by Panda's low-level I/O,
-        because it isn't supported by the standard C++ library. """
-        raise NotImplementedError
-
-    def write(self, str):
+    def write(self, b):
         if not self.__writer:
             if not self.__reader:
                 # The stream is not even open at all.
-                message = 'I/O operation on closed file'
-                raise ValueError, message
+                raise ValueError("I/O operation on closed file")
+
             # The stream is open only in read mode.
-            message = 'Attempt to write to read-only stream'
-            raise IOError, message
+            raise IOError("Attempt to write to read-only stream")
 
         self.__stream.clear()  # clear eof flag
-        self.__writer.appendData(str)
+        self.__writer.appendData(b)
         self.__lastWrite = True
 
     def writelines(self, lines):
         if not self.__writer:
             if not self.__reader:
                 # The stream is not even open at all.
-                message = 'I/O operation on closed file'
-                raise ValueError, message
+                raise ValueError("I/O operation on closed file")
+
             # The stream is open only in read mode.
-            message = 'Attempt to write to read-only stream'
-            raise IOError, message
+            raise IOError("Attempt to write to read-only stream")
 
         self.__stream.clear()  # clear eof flag
         for line in lines:
             self.__writer.appendData(line)
         self.__lastWrite = True
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, t, v, tb):
-        self.close()
-
-open = file
 
 def listdir(path):
     """ Implements os.listdir over vfs. """
     files = []
     dirlist = _vfs.scanDirectory(core.Filename.fromOsSpecific(path))
     if dirlist is None:
-        message = 'No such file or directory: %s' % (path)
-        raise OSError, message
+        raise OSError("No such file or directory: '%s'" % (path))
+
     for file in dirlist:
         files.append(file.getFilename().getBasename())
     return files

dtool/src/prc/streamReader.cxx

@@ -112,25 +112,6 @@ skip_bytes(size_t size) {
   }
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: StreamReader::extract_bytes
-//       Access: Published
-//  Description: Extracts the indicated number of bytes in the
-//               stream and returns them as a string.  Returns empty
-//               string at end-of-file.
-////////////////////////////////////////////////////////////////////
-string StreamReader::
-extract_bytes(size_t size) {
-  if (_in->eof() || _in->fail()) {
-    return string();
-  }
-
-  char *buffer = (char *)alloca(size);
-  _in->read(buffer, size);
-  size_t read_bytes = _in->gcount();
-  return string(buffer, read_bytes);
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: StreamReader::extract_bytes
 //       Access: Published
@@ -150,9 +131,28 @@ extract_bytes(unsigned char *into, size_t size) {
   return _in->gcount();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: StreamReader::extract_bytes
+//       Access: Public
+//  Description: Extracts the indicated number of bytes in the
+//               stream and returns them as a string.  Returns empty
+//               string at end-of-file.
+////////////////////////////////////////////////////////////////////
+string StreamReader::
+extract_bytes(size_t size) {
+  if (_in->eof() || _in->fail()) {
+    return string();
+  }
+
+  char *buffer = (char *)alloca(size);
+  _in->read(buffer, size);
+  size_t read_bytes = _in->gcount();
+  return string(buffer, read_bytes);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: StreamReader::readline
-//       Access: Published
+//       Access: Public
 //  Description: Assumes the stream represents a text file, and
 //               extracts one line up to and including the trailing
 //               newline character.  Returns empty string when the end
@@ -177,4 +177,3 @@ readline() {
 
   return line;
 }
-

dtool/src/prc/streamReader.h

@@ -66,12 +66,16 @@ PUBLISHED:
   BLOCKING string get_fixed_string(size_t size);
 
   BLOCKING void skip_bytes(size_t size);
-  BLOCKING string extract_bytes(size_t size);
   BLOCKING size_t extract_bytes(unsigned char *into, size_t size);
+  EXTENSION(BLOCKING PyObject *extract_bytes(size_t size));
 
-  BLOCKING string readline();
+  EXTENSION(BLOCKING PyObject *readline());
   EXTENSION(BLOCKING PyObject *readlines());
 
+public:
+  BLOCKING string extract_bytes(size_t size);
+  BLOCKING string readline();
+
 private:
   istream *_in;
   bool _owns_stream;

makepanda/makepanda.py

@@ -381,9 +381,10 @@ if (RUNTIME or RTDIST):
     if (RUNTIME):
         outputdir_suffix += "_rt"
 
-    RTDIST_VERSION = DISTRIBUTOR.strip() + "_" + MAJOR_VERSION
-elif (DISTRIBUTOR == ""):
+if DISTRIBUTOR == "":
     DISTRIBUTOR = "makepanda"
+else:
+    RTDIST_VERSION = DISTRIBUTOR.strip() + "_" + MAJOR_VERSION
 
 if not IsCustomOutputDir():
     if GetTarget() == "windows" and GetTargetArch() == 'x64':
@@ -1800,10 +1801,25 @@ def FreezePy(target, inputs, opts):
     if sys.version_info >= (2, 6):
         cmdstr += "-B "
 
-    cmdstr += os.path.join("direct", "src", "showutil", "pfreeze.py")
-    src = inputs.pop(0)
+    cmdstr += os.path.join(GetOutputDir(), "direct", "showutil", "pfreeze.py")
+
+    if 'FREEZE_STARTUP' in opts:
+        cmdstr += " -s"
+
+    if GetOrigExt(target) == '.exe':
+        src = inputs.pop(0)
+    else:
+        src = ""
+
     for i in inputs:
-      cmdstr += " -i " + os.path.splitext(i)[0]
+        i = os.path.splitext(i)[0]
+        i = i.replace('/', '.')
+
+        if i.startswith('direct.src'):
+            i = i.replace('.src.', '.')
+
+        cmdstr += " -i " + i
+
     cmdstr += " -o " + target + " " + src
 
     if ("LINK_PYTHON_STATIC" in opts):
@@ -1919,6 +1935,7 @@ def CompileAnything(target, inputs, opts, progress = None):
         exit("No input files for target "+target)
     infile = inputs[0]
     origsuffix = GetOrigExt(target)
+
     if (len(inputs) == 1 and origsuffix == GetOrigExt(infile)):
         # It must be a simple copy operation.
         ProgressOutput(progress, "Copying file", target)
@@ -1926,15 +1943,26 @@ def CompileAnything(target, inputs, opts, progress = None):
         if (origsuffix==".exe" and GetHost() != "windows"):
             os.system("chmod +x \"%s\"" % target)
         return
+
     elif (target.endswith(".py")):
         ProgressOutput(progress, "Generating", target)
         return GenPyExtensions(target, inputs, opts)
+
     elif (infile.endswith(".py")):
-        if (origsuffix==".exe"):
+        if origsuffix == ".obj":
+            source = os.path.splitext(target)[0] + ".c"
+            SetOrigExt(source, ".c")
+            ProgressOutput(progress, "Building frozen source", source)
+            FreezePy(source, inputs, opts)
+            ProgressOutput(progress, "Building C++ object", target)
+            return CompileCxx(target, source, opts)
+
+        if origsuffix == ".exe":
             ProgressOutput(progress, "Building frozen executable", target)
         else:
             ProgressOutput(progress, "Building frozen library", target)
         return FreezePy(target, inputs, opts)
+
     elif (infile.endswith(".idl")):
         ProgressOutput(progress, "Compiling MIDL file", infile)
         return CompileMIDL(target, infile, opts)
@@ -5021,10 +5049,12 @@ if (RTDIST or RUNTIME):
     TargetAdd("libp3d_plugin_static.ilb", input='plugin_get_twirl_data.obj')
 
   if (PkgSkip("PYTHON")==0 and RTDIST):
+    TargetAdd('p3dpython_frozen.obj', opts=['DIR:direct/src/showbase', 'FREEZE_STARTUP'], input='VFSImporter.py')
     TargetAdd('p3dpython_p3dpython_composite1.obj', opts=OPTS, input='p3dpython_composite1.cxx')
     TargetAdd('p3dpython_p3dPythonMain.obj', opts=OPTS, input='p3dPythonMain.cxx')
     TargetAdd('p3dpython.exe', input='p3dpython_p3dpython_composite1.obj')
     TargetAdd('p3dpython.exe', input='p3dpython_p3dPythonMain.obj')
+    TargetAdd('p3dpython.exe', input='p3dpython_frozen.obj')
     TargetAdd('p3dpython.exe', input=COMMON_PANDA_LIBS)
     TargetAdd('p3dpython.exe', input='libp3tinyxml.ilb')
     TargetAdd('p3dpython.exe', input='libp3interrogatedb.dll')

panda/src/express/p3express_ext_composite.cxx

@@ -5,3 +5,4 @@
 #include "streamReader_ext.cxx"
 #include "typeHandle_ext.cxx"
 #include "virtualFileSystem_ext.cxx"
+#include "virtualFile_ext.cxx"

panda/src/express/ramfile.h

@@ -18,6 +18,7 @@
 #include "pandabase.h"
 #include "typedef.h"
 #include "referenceCount.h"
+#include "extension.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : Ramfile
@@ -30,17 +31,23 @@ PUBLISHED:
 
   INLINE void seek(size_t pos);
   INLINE size_t tell() const;
-  string read(size_t length);
-  string readline();
+  EXTENSION(PyObject *read(size_t length));
+  EXTENSION(PyObject *readline());
   EXTENSION(PyObject *readlines());
 
-  INLINE const string &get_data() const;
+  EXTENSION(PyObject *get_data() const);
   INLINE size_t get_data_size() const;
   INLINE void clear();
 
 public:
+  string read(size_t length);
+  string readline();
+  INLINE const string &get_data() const;
+
   size_t _pos;
   string _data;
+
+  friend class Extension<Ramfile>;
 };
 
 #include "ramfile.I"

panda/src/express/ramfile_ext.cxx

@@ -16,6 +16,49 @@
 
 #ifdef HAVE_PYTHON
 
+////////////////////////////////////////////////////////////////////
+//     Function: Ramfile::read
+//       Access: Published
+//  Description: Extracts the indicated number of bytes in the
+//               stream and returns them as a string (or bytes,
+//               in Python 3).  Returns empty string at end-of-file.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<Ramfile>::
+read(size_t length) {
+  size_t data_length = _this->get_data_size();
+  const char *data = _this->_data.data() + _this->_pos;
+  length = min(length, data_length - _this->_pos);
+  _this->_pos = min(_this->_pos + length, data_length);
+
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize((char *)data, length);
+#else
+  return PyString_FromStringAndSize((char *)data, length);
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Ramfile::readline
+//       Access: Published
+//  Description: Assumes the stream represents a text file, and
+//               extracts one line up to and including the trailing
+//               newline character.  Returns empty string when the end
+//               of file is reached.
+//
+//               The interface here is intentionally designed to be
+//               similar to that for Python's File.readline()
+//               function.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<Ramfile>::
+readline() {
+  string line = _this->readline();
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize(line.data(), line.size());
+#else
+  return PyString_FromStringAndSize(line.data(), line.size());
+#endif
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: Ramfile::readlines
 //       Access: Published
@@ -44,5 +87,19 @@ readlines() {
   return lst;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: Ramfile::get_data
+//       Access: Published
+//  Description: Returns the entire buffer contents as a string,
+//               regardless of the current data pointer.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<Ramfile>::
+get_data() const {
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize(_this->_data.data(), _this->_data.size());
+#else
+  return PyString_FromStringAndSize(_this->_data.data(), _this->_data.size());
 #endif
+}
 
+#endif

panda/src/express/ramfile_ext.h

@@ -32,7 +32,11 @@
 template<>
 class Extension<Ramfile> : public ExtensionBase<Ramfile> {
 public:
-  BLOCKING PyObject *readlines();
+  PyObject *read(size_t length);
+  PyObject *readline();
+  PyObject *readlines();
+
+  PyObject *get_data() const;
 };
 
 #endif  // HAVE_PYTHON

panda/src/express/streamReader_ext.cxx

@@ -16,6 +16,59 @@
 
 #ifdef HAVE_PYTHON
 
+////////////////////////////////////////////////////////////////////
+//     Function: StreamReader::extract_bytes
+//       Access: Published
+//  Description: Extracts the indicated number of bytes in the
+//               stream and returns them as a string (or bytes,
+//               in Python 3).  Returns empty string at end-of-file.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<StreamReader>::
+extract_bytes(size_t size) {
+  unsigned char *buffer = (unsigned char *)alloca(size);
+  size_t read_bytes = _this->extract_bytes(buffer, size);
+
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize((char *)buffer, read_bytes);
+#else
+  return PyString_FromStringAndSize((char *)buffer, read_bytes);
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extension<StreamReader>::readline
+//       Access: Published
+//  Description: Assumes the stream represents a text file, and
+//               extracts one line up to and including the trailing
+//               newline character.  Returns empty string when the end
+//               of file is reached.
+//
+//               The interface here is intentionally designed to be
+//               similar to that for Python's File.readline()
+//               function.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<StreamReader>::
+readline() {
+  istream *in = _this->get_istream();
+
+  string line;
+  int ch = in->get();
+  while (!in->eof() && !in->fail()) {
+    line += ch;
+    if (ch == '\n') {
+      // Here's the newline character.
+      break;
+    }
+    ch = in->get();
+  }
+
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize(line.data(), line.size());
+#else
+  return PyString_FromStringAndSize(line.data(), line.size());
+#endif
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: StreamReader::readlines
 //       Access: Published
@@ -29,16 +82,17 @@ readlines() {
     return NULL;
   }
 
-  string line = _this->readline();
-  while (!line.empty()) {
+  PyObject *py_line = readline();
+
 #if PY_MAJOR_VERSION >= 3
-    PyObject *py_line = PyBytes_FromStringAndSize(line.data(), line.size());
+  while (PyBytes_GET_SIZE(py_line) > 0) {
 #else
-    PyObject *py_line = PyString_FromStringAndSize(line.data(), line.size());
+  while (PyString_GET_SIZE(py_line) > 0) {
 #endif
-
     PyList_Append(lst, py_line);
     Py_DECREF(py_line);
+
+    py_line = readline();
   }
 
   return lst;

panda/src/express/streamReader_ext.h

@@ -32,7 +32,10 @@
 template<>
 class Extension<StreamReader> : public ExtensionBase<StreamReader> {
 public:
+  BLOCKING PyObject *extract_bytes(size_t size);
+  BLOCKING PyObject *readline();
   BLOCKING PyObject *readlines();
+  BLOCKING PyObject *get_data() const;
 };
 
 #endif  // HAVE_PYTHON

panda/src/express/virtualFile.h

@@ -58,12 +58,12 @@ PUBLISHED:
   BLOCKING void ls(ostream &out = cout) const;
   BLOCKING void ls_all(ostream &out = cout) const;
 
-  BLOCKING INLINE string read_file(bool auto_unwrap) const;
+  EXTENSION(BLOCKING PyObject *read_file(bool auto_unwrap) const);
   BLOCKING virtual istream *open_read_file(bool auto_unwrap) const;
   BLOCKING virtual void close_read_file(istream *stream) const;
   virtual bool was_read_successful() const;
 
-  BLOCKING INLINE bool write_file(const string &data, bool auto_wrap);
+  EXTENSION(BLOCKING PyObject *write_file(PyObject *data, bool auto_wrap));
   BLOCKING virtual ostream *open_write_file(bool auto_wrap, bool truncate);
   BLOCKING virtual ostream *open_append_file();
   BLOCKING virtual void close_write_file(ostream *stream);
@@ -82,6 +82,9 @@ public:
   virtual bool atomic_compare_and_exchange_contents(string &orig_contents, const string &old_contents, const string &new_contents);
   virtual bool atomic_read_contents(string &contents) const;
 
+  INLINE string read_file(bool auto_unwrap) const;
+  INLINE bool write_file(const string &data, bool auto_wrap);
+
   INLINE void set_original_filename(const Filename &filename);
   bool read_file(string &result, bool auto_unwrap) const;
   virtual bool read_file(pvector<unsigned char> &result, bool auto_unwrap) const;

panda/src/express/virtualFile_ext.cxx

@@ -0,0 +1,83 @@
+// Filename: virtualFile_ext.cxx
+// Created by:  rdb (15Sep15)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "virtualFile_ext.h"
+#include "vector_uchar.h"
+
+#ifdef HAVE_PYTHON
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFile::read_file
+//       Access: Published
+//  Description: Convenience function; returns the entire contents of
+//               the indicated file as a string (or as a bytes object,
+//               in Python 3).
+//
+//               This variant on read_file() is implemented directly
+//               for Python, as a small optimization, to avoid the
+//               double-construction of a string object that would be
+//               otherwise required for the return value.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<VirtualFile>::
+read_file(bool auto_unwrap) const {
+  vector_uchar pv;
+  bool okflag = _this->read_file(pv, auto_unwrap);
+  nassertr(okflag, NULL);
+
+#if PY_MAJOR_VERSION >= 3
+  if (pv.empty()) {
+    return PyBytes_FromStringAndSize("", 0);
+  } else {
+    return PyBytes_FromStringAndSize((const char *)&pv[0], pv.size());
+  }
+#else
+  if (pv.empty()) {
+    return PyString_FromStringAndSize("", 0);
+  } else {
+    return PyString_FromStringAndSize((const char *)&pv[0], pv.size());
+  }
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFile::write_file
+//       Access: Published
+//  Description: Convenience function; writes the entire contents of
+//               the indicated file as a string.
+//
+//               This variant on write_file() is implemented directly
+//               for Python, as a small optimization, to avoid the
+//               double-construction of a string object that would be
+//               otherwise required.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<VirtualFile>::
+write_file(PyObject *data, bool auto_wrap) {
+  char *buffer;
+  Py_ssize_t length;
+
+#if PY_MAJOR_VERSION >= 3
+  if (PyBytes_AsStringAndSize(data, &buffer, &length) == -1) {
+    return NULL;
+  }
+#else
+  if (PyString_AsStringAndSize(data, &buffer, &length) == -1) {
+    return NULL;
+  }
+#endif
+
+  bool result = _this->write_file((const unsigned char *)buffer, length, auto_wrap);
+  return PyBool_FromLong(result);
+}
+
+#endif  // HAVE_PYTHON

panda/src/express/virtualFile_ext.h

@@ -0,0 +1,42 @@
+// Filename: virtualFile_ext.h
+// Created by:  rdb (15Sep15)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef VIRTUALFILE_EXT_H
+#define VIRTUALFILE_EXT_H
+
+#include "dtoolbase.h"
+
+#ifdef HAVE_PYTHON
+
+#include "extension.h"
+#include "virtualFile.h"
+#include "py_panda.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : Extension<VirtualFile>
+// Description : This class defines the extension methods for
+//               VirtualFile, which are called instead of
+//               any C++ methods with the same prototype.
+////////////////////////////////////////////////////////////////////
+template<>
+class Extension<VirtualFile> : public ExtensionBase<VirtualFile> {
+public:
+  PyObject *read_file(bool auto_unwrap) const;
+  PyObject *write_file(PyObject *data, bool auto_wrap);
+};
+
+#endif  // HAVE_PYTHON
+
+#endif  // VIRTUALFILE_EXT_H
+