automatic package management

direct/src/p3d/AppRunner.py

@@ -12,6 +12,7 @@ runp3d.py for a command-line tool to invoke this module.
 import sys
 import os
 import types
+import shutil
 import __builtin__
 
 if 'VFSImporter' in sys.modules:
@@ -33,7 +34,7 @@ else:
     from direct.showbase import VFSImporter
 
 from direct.showbase.DirectObject import DirectObject
-from pandac.PandaModules import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, Thread, WindowProperties, ExecutionEnvironment, PandaSystem, Notify, StreamWriter, ConfigVariableString, initAppForGui
+from pandac.PandaModules import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, Thread, WindowProperties, ExecutionEnvironment, PandaSystem, Notify, StreamWriter, ConfigVariableString, initAppForGui, TiXmlDocument
 from pandac import PandaModules
 from direct.stdpy import file, glob
 from direct.task.TaskManagerGlobal import taskMgr
@@ -41,6 +42,9 @@ from direct.showbase.MessengerGlobal import messenger
 from direct.showbase import AppRunnerGlobal
 from direct.directnotify.DirectNotifyGlobal import directNotify
 from direct.p3d.HostInfo import HostInfo
+from direct.p3d.ScanDirectoryNode import ScanDirectoryNode
+from direct.p3d.InstalledHostData import InstalledHostData
+from direct.p3d.InstalledPackageData import InstalledPackageData
 
 # These imports are read by the C++ wrapper in p3dPythonRun.cxx.
 from direct.p3d.JavaScript import UndefinedObject, Undefined, ConcreteStruct, BrowserObject
@@ -66,6 +70,11 @@ class AppRunner(DirectObject):
     development purposes.  """
 
     notify = directNotify.newCategory("AppRunner")
+
+    ConfigBasename = 'config.xml'
+
+    # Default values for parameters that are absent from the config file:
+    maxDiskUsage = 1073741824  # 1 GB
     
     def __init__(self):
         DirectObject.__init__(self)
@@ -344,6 +353,63 @@ class AppRunner(DirectObject):
             self.hosts[hostUrl] = host
         return host
 
+    def getHostWithDir(self, hostDir):
+        """ Returns the HostInfo object that corresponds to the
+        indicated on-disk host directory.  This would be used when
+        reading a host directory from disk, instead of downloading it
+        from a server.  Supply the full path to the host directory, as
+        a Filename.  Returns None if the contents.xml in the indicated
+        host directory cannot be read or doesn't seem consistent. """
+
+        host = HostInfo(None, hostDir = hostDir, appRunner = self)
+        if not host.readContentsFile():
+            # Couldn't read the contents.xml file
+            return None
+
+        if not host.hostUrl:
+            # The contents.xml file there didn't seem to indicate the
+            # same host directory.
+            return None
+
+        host2 = self.hosts.get(host.hostUrl)
+        if host2 is None:
+            # No such host already; store this one.
+            self.hosts[host.hostUrl] = host
+            return host
+
+        if host2.hostDir != host.hostDir:
+            # Hmm, we already have that host somewhere else.
+            return None
+
+        # We already have that host, and it's consistent.
+        return host2
+
+    def deletePackages(self, packages):
+        """ Removes all of the indicated packages from the disk,
+        uninstalling them and deleting all of their files.  The
+        packages parameter must be a list of one or more PackageInfo
+        objects, for instance as returned by getHost().getPackage().
+        Returns the list of packages that were NOT found. """
+
+        for hostUrl, host in self.hosts.items():
+            packages = host.deletePackages(packages)
+
+            if not host.packages:
+                # If that's all of the packages for this host, delete
+                # the host directory too.
+                del self.hosts[hostUrl]
+                self.__deleteHostFiles(host)
+                
+        return packages
+
+    def __deleteHostFiles(self, host):
+        """ Called by deletePackages(), this removes all the files for
+        the indicated host (for which we have presumably already
+        removed all of the packages). """
+
+        self.notify.info("Deleting host %s: %s" % (host.hostUrl, host.hostDir))
+        shutil.rmtree(host.hostDir.toOsSpecific(), True)
+
     def freshenFile(self, host, fileSpec, localPathname):
         """ Ensures that the localPathname is the most current version
         of the file defined by fileSpec, as offered by host.  If not,
@@ -392,6 +458,114 @@ class AppRunner(DirectObject):
 
         return True
 
+    def scanInstalledPackages(self):
+        """ Scans the hosts and packages already installed locally on
+        the system.  Returns a list of InstalledHostData objects, each
+        of which contains a list of InstalledPackageData objects. """
+
+        result = []
+        hostsFilename = Filename(self.rootDir, 'hosts')
+        hostsDir = ScanDirectoryNode(hostsFilename)
+        for dirnode in hostsDir.nested:
+            host = self.getHostWithDir(dirnode.pathname)
+            hostData = InstalledHostData(host, dirnode)
+
+            if host:
+                for package in host.getAllPackages():
+                    packageDir = package.getPackageDir()
+                    if not packageDir.exists():
+                        continue
+
+                    subdir = dirnode.extractSubdir(packageDir)
+                    if not subdir:
+                        # This package, while defined by the host, isn't installed
+                        # locally; ignore it.
+                        continue
+
+                    packageData = InstalledPackageData(package, subdir)
+                    hostData.packages.append(packageData)
+
+            # Now that we've examined all of the packages for the host,
+            # anything left over is junk.
+            for subdir in dirnode.nested:
+                packageData = InstalledPackageData(None, subdir)
+                hostData.packages.append(packageData)
+
+            result.append(hostData)
+
+        return result
+
+    def readConfigXml(self):
+        """ Reads the config.xml file that may be present in the root
+        directory. """
+
+        if not hasattr(PandaModules, 'TiXmlDocument'):
+            return
+
+        filename = Filename(self.rootDir, self.ConfigBasename)
+        doc = TiXmlDocument(filename.toOsSpecific())
+        if not doc.LoadFile():
+            return
+
+        xconfig = doc.FirstChildElement('config')
+        if xconfig:
+            maxDiskUsage = xconfig.Attribute('max_disk_usage')
+            try:
+                self.maxDiskUsage = int(maxDiskUsage or '')
+            except ValueError:
+                pass
+            
+
+    def checkDiskUsage(self):
+        """ Checks the total disk space used by all packages, and
+        removes old packages if necessary. """
+
+        totalSize = 0
+        hosts = self.scanInstalledPackages()
+        for hostData in hosts:
+            for packageData in hostData.packages:
+                totalSize += packageData.totalSize
+        self.notify.info("Total Panda3D disk space used: %s MB" % (
+            (totalSize + 524288) / 1048576))
+        self.notify.info("Configured max usage is: %s MB" % (
+            (self.maxDiskUsage + 524288) / 1048576))
+        if totalSize <= self.maxDiskUsage:
+            # Still within budget; no need to clean up anything.
+            return
+
+        # OK, we're over budget.  Now we have to remove old packages.
+        usedPackages = []
+        for hostData in hosts:
+            for packageData in hostData.packages:
+                if packageData.package and packageData.package.installed:
+                    # Don't uninstall any packages we're currently using.
+                    continue
+                
+                usedPackages.append((packageData.lastUse, packageData))
+
+        # Sort the packages into oldest-first order.
+        usedPackages.sort()
+
+        # Delete packages until we free up enough space.
+        packages = []
+        for lastUse, packageData in usedPackages:
+            if totalSize <= self.maxDiskUsage:
+                break
+            totalSize -= packageData.totalSize
+
+            if packageData.package:
+                packages.append(packageData.package)
+            else:
+                # If it's an unknown package, just delete it directly.
+                print "Deleting unknown package %s" % (packageData.pathname)
+                shutil.rmtree(packageData.pathname.toOsSpecific(), True)
+
+        packages = self.deletePackages(packages)
+        if packages:
+            print "Unable to delete %s packages" % (len(packages))
+        
+        return
+
     def stop(self):
         """ This method can be called by JavaScript to stop the
         application. """
@@ -473,6 +647,8 @@ class AppRunner(DirectObject):
             os.path.getsize = file.getsize
             sys.modules['glob'] = glob
 
+        self.checkDiskUsage()
+
     def __startIfReady(self):
         """ Called internally to start the application. """
         if self.started:
@@ -557,6 +733,10 @@ class AppRunner(DirectObject):
         # The "super mirror" URL, generally used only by panda3d.exe.
         self.superMirrorUrl = superMirrorUrl
 
+        # Now that we have rootDir, we can read the config file.
+        self.readConfigXml()
+        
+
     def addPackageInfo(self, name, platform, version, hostUrl, hostDir = None):
         """ Called by the browser for each one of the "required"
         packages that were preloaded before starting the application.

direct/src/p3d/FileSpec.py

@@ -122,7 +122,7 @@ class FileSpec:
             return False
 
         if notify:
-            notify.debug("hash check ok: %s" % (pathname))
+            notify.info("hash check ok: %s" % (pathname))
 
         # The hash is OK after all.  Change the file's timestamp back
         # to what we expect it to be, so we can quick-verify it
@@ -164,7 +164,7 @@ class FileSpec:
             return False
 
         if notify:
-            notify.debug("hash check ok: %s" % (pathname))
+            notify.info("hash check ok: %s" % (pathname))
 
         # The hash is OK.  If the timestamp is wrong, change it back
         # to what we expect it to be, so we can quick-verify it

direct/src/p3d/HostInfo.py

@@ -4,6 +4,7 @@ from direct.p3d.PackageInfo import PackageInfo
 from direct.p3d.FileSpec import FileSpec
 from direct.directnotify.DirectNotifyGlobal import directNotify
 import time
+import shutil
 
 class HostInfo:
     """ This class represents a particular download host serving up
@@ -35,8 +36,8 @@ class HostInfo:
         the default is perPlatform = True. """
         
         assert appRunner or rootDir or hostDir
-        
-        self.hostUrl = hostUrl
+
+        self.__setHostUrl(hostUrl)
         self.appRunner = appRunner
         self.rootDir = rootDir
         if rootDir is None and appRunner:
@@ -48,18 +49,6 @@ class HostInfo:
         if perPlatform is None:
             self.perPlatform = asMirror
 
-        # hostUrlPrefix is the host URL, but it is guaranteed to end
-        # with a slash.
-        self.hostUrlPrefix = hostUrl
-        if self.hostUrlPrefix[-1] != '/':
-            self.hostUrlPrefix += '/'
-
-        # downloadUrlPrefix is the URL prefix that should be used for
-        # everything other than the contents.xml file.  It might be
-        # the same as hostUrlPrefix, but in the case of an
-        # https-protected hostUrl, it will be the cleartext channel.
-        self.downloadUrlPrefix = self.hostUrlPrefix
-
         # Initially false, this is set true when the contents file is
         # successfully read.
         self.hasContentsFile = False
@@ -82,6 +71,27 @@ class HostInfo:
         # will be filled in when the contents file is read.
         self.packages = {}
 
+    def __setHostUrl(self, hostUrl):
+        """ Assigns self.hostUrl, and related values. """
+        self.hostUrl = hostUrl
+
+        if self.hostUrl is None:
+            # A special case: the URL will be set later.
+            self.hostUrlPrefix = None
+            self.downloadUrlPrefix = None
+        else:
+            # hostUrlPrefix is the host URL, but it is guaranteed to end
+            # with a slash.
+            self.hostUrlPrefix = hostUrl
+            if self.hostUrlPrefix[-1] != '/':
+                self.hostUrlPrefix += '/'
+
+            # downloadUrlPrefix is the URL prefix that should be used for
+            # everything other than the contents.xml file.  It might be
+            # the same as hostUrlPrefix, but in the case of an
+            # https-protected hostUrl, it will be the cleartext channel.
+            self.downloadUrlPrefix = self.hostUrlPrefix
+
     def downloadContentsFile(self, http, redownload = False):
         """ Downloads the contents.xml file for this particular host,
         synchronously, and then reads it.  Returns true on success,
@@ -213,10 +223,14 @@ class HostInfo:
             return False
 
         # Look for our own entry in the hosts table.
-        self.__findHostXml(xcontents)
+        if self.hostUrl:
+            self.__findHostXml(xcontents)
+        else:
+            assert self.hostDir
+            self.__findHostXmlForHostDir(xcontents)
 
         if not self.hostDir:
-            self.__determineHostDir(None)
+            self.hostDir = self.__determineHostDir(None, self.hostUrl)
 
         # Get the list of packages available for download and/or import.
         xpackage = xcontents.FirstChildElement('package')
@@ -275,6 +289,36 @@ class HostInfo:
             
             xhost = xhost.NextSiblingElement('host')
 
+    def __findHostXmlForHostDir(self, xcontents):
+        """ Looks for the <host> or <alt_host> entry in the
+        contents.xml that corresponds to the host dir that we read the
+        contents.xml from.  This is used when reading a contents.xml
+        file found on disk, as opposed to downloading it from a
+        site. """
+        
+        xhost = xcontents.FirstChildElement('host')
+        while xhost:
+            url = xhost.Attribute('url')
+            hostDirBasename = xhost.Attribute('host_dir')
+            hostDir = self.__determineHostDir(hostDirBasename, url)
+            if hostDir == self.hostDir:
+                self.__setHostUrl(url)
+                self.readHostXml(xhost)
+                return
+
+            xalthost = xhost.FirstChildElement('alt_host')
+            while xalthost:
+                url = xalthost.Attribute('url')
+                hostDirBasename = xalthost.Attribute('host_dir')
+                hostDir = self.__determineHostDir(hostDirBasename, url)
+                if hostDir == self.hostDir:
+                    self.__setHostUrl(url)
+                    self.readHostXml(xalthost)
+                    return
+                xalthost = xalthost.NextSiblingElement('alt_host')
+            
+            xhost = xhost.NextSiblingElement('host')
+
     def readHostXml(self, xhost):
         """ Reads a <host> or <alt_host> entry and applies the data to
         this object. """
@@ -284,7 +328,8 @@ class HostInfo:
             self.descriptiveName = descriptiveName
 
         hostDirBasename = xhost.Attribute('host_dir')
-        self.__determineHostDir(hostDirBasename)
+        if not self.hostDir:
+            self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl)
 
         # Get the "download" URL, which is the source from which we
         # download everything other than the contents.xml file.
@@ -381,29 +426,85 @@ class HostInfo:
 
         return packages
 
-    def __determineHostDir(self, hostDirBasename):
+    def getAllPackages(self):
+        """ Returns a list of all available packages provided by this
+        host. """
+
+        result = []
+
+        items = self.packages.items()
+        items.sort()
+        for key, platforms in items:
+            if self.perPlatform:
+                # If we maintain a different answer per platform,
+                # return all of them.
+                pitems = platforms.items()
+                pitems.sort()
+                for pkey, package in pitems:
+                    result.append(package)
+            else:
+                # If we maintain a a host for the current platform
+                # only (e.g. a client copy), then return only the
+                # current platform, or no particular platform.
+                package = platforms.get(PandaSystem.getPlatform(), None)
+                if not package:
+                    package = platforms.get(None, None)
+
+                if package:
+                    result.append(package)
+
+        return result
+
+    def deletePackages(self, packages):
+        """ Removes all of the indicated packages from the disk,
+        uninstalling them and deleting all of their files.  The
+        packages parameter must be a list of one or more PackageInfo
+        objects, for instance as returned by getPackage().  Returns
+        the list of packages that were NOT found. """
+
+        packages = packages[:]
+        
+        for key, platforms in self.packages.items():
+            for platform, package in platforms.items():
+                if package in packages:
+                    self.__deletePackageFiles(package)
+                    del platforms[platform]
+                    packages.remove(package)
+
+            if not platforms:
+                # If we've removed all the platforms for a given
+                # package, remove the key from the toplevel map.
+                del self.packages[key]
+
+        return packages
+
+    def __deletePackageFiles(self, package):
+        """ Called by deletePackage(), this actually removes the files
+        for the indicated package. """
+
+        self.notify.info("Deleting package %s: %s" % (package.packageName, package.getPackageDir()))
+        shutil.rmtree(package.getPackageDir().toOsSpecific(), True)
+
+    def __determineHostDir(self, hostDirBasename, hostUrl):
         """ Hashes the host URL into a (mostly) unique directory
         string, which will be the root of the host's install tree.
-        Stores the resulting path, as a Filename, in self.hostDir.
+        Returns the resulting path, as a Filename.
 
         This code is duplicated in C++, in
         P3DHost::determine_host_dir(). """
 
-        if self.hostDir:
-            return
-
         if hostDirBasename:
             # If the contents.xml specified a host_dir parameter, use
             # it.
-            self.hostDir = self.rootDir + '/hosts'
+            hostDir = self.rootDir.cStr() + '/hosts'
             for component in hostDirBasename.split('/'):
                 if component:
                     if component[0] == '.':
                         # Forbid ".foo" or "..".
                         component = 'x' + component
-                    self.hostDir += '/'
-                    self.hostDir += component
-            return
+                    hostDir += '/'
+                    hostDir += component
+            return Filename(hostDir)
 
         hostDir = 'hosts/'
 
@@ -414,22 +515,22 @@ class HostInfo:
         # We could use URLSpec, but we do it by hand instead, to make
         # it more likely that our hash code will exactly match the
         # similar logic in P3DHost.
-        p = self.hostUrl.find('://')
+        p = hostUrl.find('://')
         if p != -1:
             start = p + 3
-            end = self.hostUrl.find('/', start)
+            end = hostUrl.find('/', start)
             # Now start .. end is something like "username@host:port".
 
-            at = self.hostUrl.find('@', start)
+            at = hostUrl.find('@', start)
             if at != -1 and at < end:
                 start = at + 1
 
-            colon = self.hostUrl.find(':', start)
+            colon = hostUrl.find(':', start)
             if colon != -1 and colon < end:
                 end = colon
 
             # Now start .. end is just the hostname.
-            hostname = self.hostUrl[start : end]
+            hostname = hostUrl[start : end]
 
         # Now build a hash string of the whole URL.  We'll use MD5 to
         # get a pretty good hash, with a minimum chance of collision.
@@ -450,7 +551,8 @@ class HostInfo:
             keepHash = keepHash / 2;
 
         md = HashVal()
-        md.hashString(self.hostUrl)
+        md.hashString(hostUrl)
         hostDir += md.asHex()[:keepHash * 2]
 
-        self.hostDir = Filename(self.rootDir, hostDir)
+        hostDir = Filename(self.rootDir, hostDir)
+        return hostDir

direct/src/p3d/InstalledHostData.py

@@ -0,0 +1,22 @@
+from pandac.PandaModules import URLSpec
+
+class InstalledHostData:
+    """ A list of instances of this class is returned by
+    AppRunner.scanInstalledPackages().  Each of these corresponds to a
+    particular host that has provided packages that have been
+    installed on the local client. """
+    
+    def __init__(self, host, dirnode):
+        self.host = host
+        self.pathname = dirnode.pathname
+        self.totalSize = dirnode.getTotalSize()
+        self.packages = []
+
+        if self.host:
+            self.hostUrl = self.host.hostUrl
+            self.descriptiveName = self.host.descriptiveName
+            if not self.descriptiveName:
+                self.descriptiveName = URLSpec(self.hostUrl).getServer()
+        else:
+            self.hostUrl = 'unknown'
+            self.descriptiveName = 'unknown'

direct/src/p3d/InstalledPackageData.py

@@ -0,0 +1,28 @@
+class InstalledPackageData:
+    """ A list of instances of this class is maintained by
+    InstalledHostData (which is in turn returned by
+    AppRunner.scanInstalledPackages()).  Each of these corresponds to
+    a particular package that has been installed on the local
+    client. """
+
+    def __init__(self, package, dirnode):
+        self.package = package
+        self.pathname = dirnode.pathname
+        self.totalSize = dirnode.getTotalSize()
+        self.lastUse = None
+
+        if self.package:
+            self.displayName = self.package.getFormattedName()
+            xusage = self.package.getUsage()
+
+            if xusage:
+                lastUse = xusage.Attribute('last_use')
+                try:
+                    lastUse = int(lastUse or '')
+                except ValueError:
+                    lastUse = None
+                self.lastUse = lastUse
+
+        else:
+            self.displayName = dirnode.pathname.getBasename()
+            

direct/src/p3d/PackageInfo.py

@@ -1,6 +1,7 @@
-from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector
+from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector, TiXmlDocument, TiXmlDeclaration, TiXmlElement
 from pandac import PandaModules
 from direct.p3d.FileSpec import FileSpec
+from direct.p3d.ScanDirectoryNode import ScanDirectoryNode
 from direct.showbase import VFSImporter
 from direct.directnotify.DirectNotifyGlobal import directNotify
 from direct.task.TaskManagerGlobal import taskMgr
@@ -8,6 +9,7 @@ import os
 import sys
 import random
 import time
+import copy
 
 class PackageInfo:
 
@@ -32,6 +34,8 @@ class PackageInfo:
     restartDownload = 3
     stepContinue = 4
 
+    UsageBasename = 'usage.xml'
+
     class InstallStep:
         """ This class is one step of the installPlan list; it
         represents a single atomic piece of the installation step, and
@@ -117,6 +121,11 @@ class PackageInfo:
         # meaning it's been added to the paths and all.
         self.installed = False
 
+        # This is set true when the package has been updated in this
+        # session, but not yet written to usage.xml.
+        self.updated = False
+        self.diskSpace = None
+
     def getPackageDir(self):
         """ Returns the directory in which this package is installed.
         This may not be known until the host's contents.xml file has
@@ -493,6 +502,7 @@ class PackageInfo:
         contents = self.__scanDirectoryRecursively(self.getPackageDir()) 
         self.__removeFileFromList(contents, self.descFileBasename)
         self.__removeFileFromList(contents, self.compressedArchive.filename)
+        self.__removeFileFromList(contents, self.UsageBasename)
         if not self.asMirror:
             self.__removeFileFromList(contents, self.uncompressedArchive.filename)
             for file in self.extracts:
@@ -507,6 +517,7 @@ class PackageInfo:
             self.notify.info("Removing %s" % (filename))
             pathname = Filename(self.getPackageDir(), filename)
             pathname.unlink()
+            self.updated = True
 
         if self.asMirror:
             return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify)
@@ -681,6 +692,8 @@ class PackageInfo:
         packageDir.  Yields one of stepComplete, stepFailed, 
         restartDownload, or stepContinue. """
 
+        self.updated = True
+
         if not urlbase:
             urlbase = self.descFileDirname + '/' + fileSpec.filename
 
@@ -817,6 +830,8 @@ class PackageInfo:
         operation.  Yields one of stepComplete, stepFailed, 
         restartDownload, or stepContinue. """
 
+        self.updated = True
+
         origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
         patchPathname = Filename(self.getPackageDir(), patchfile.file.filename)
         result = Filename.temporary('', 'patch_')
@@ -857,6 +872,8 @@ class PackageInfo:
         archive.  Yields one of stepComplete, stepFailed, 
         restartDownload, or stepContinue. """
 
+        self.updated = True
+
         sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename)
         targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
         targetPathname.unlink()
@@ -905,6 +922,8 @@ class PackageInfo:
             self.hasPackage = True
             yield self.stepComplete; return
 
+        self.updated = True
+
         mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
         self.notify.info("Unpacking %s" % (mfPathname))
         mf = Multifile()
@@ -1026,4 +1045,94 @@ class PackageInfo:
         self.installed = True
         appRunner.installedPackages.append(self)
 
+        self.markUsed()
+
         return True
+
+    def __measureDiskSpace(self):
+        """ Returns the amount of space used by this package, in
+        bytes, as determined by examining the actual contents of the
+        package directory and its subdirectories. """
+
+        thisDir = ScanDirectoryNode(self.getPackageDir(), ignoreUsageXml = True)
+        diskSpace = thisDir.getTotalSize()
+        self.notify.info("Package %s uses %s MB" % (
+            self.packageName, (diskSpace + 524288) / 1048576))
+        return diskSpace
+
+    def markUsed(self):
+        """ Marks the package as having been used.  This is normally
+        called automatically by installPackage(). """
+
+        if not hasattr(PandaModules, 'TiXmlDocument'):
+            return
+
+        if self.updated:
+            # If we've just installed a new version of the package,
+            # re-measure the actual disk space used.
+            self.diskSpace = self.__measureDiskSpace()
+
+        filename = Filename(self.getPackageDir(), self.UsageBasename)
+        doc = TiXmlDocument(filename.toOsSpecific())
+        if not doc.LoadFile():
+            decl = TiXmlDeclaration("1.0", "utf-8", "")
+            doc.InsertEndChild(decl)
+            
+        xusage = doc.FirstChildElement('usage')
+        if not xusage:
+            doc.InsertEndChild(TiXmlElement('usage'))
+            xusage = doc.FirstChildElement('usage')
+
+        now = int(time.time())
+        
+        count = xusage.Attribute('count_app')
+        try:
+            count = int(count or '')
+        except ValueError:
+            count = 0
+            xusage.SetAttribute('first_use', str(now))
+        count += 1
+        xusage.SetAttribute('count_app', str(count))
+
+        xusage.SetAttribute('last_use', str(now))
+
+        if self.updated:
+            xusage.SetAttribute('last_update', str(now))
+            self.updated = False
+        else:
+            # Since we haven't changed the disk space, we can just
+            # read it from the previous xml file.
+            diskSpace = xusage.Attribute('disk_space')
+            try:
+                diskSpace = int(diskSpace or '')
+            except ValueError:
+                # Unless it wasn't set already.
+                self.diskSpace = self.__measureDiskSpace()
+
+        xusage.SetAttribute('disk_space', str(self.diskSpace))
+
+        # Write the file to a temporary filename, then atomically move
+        # it to its actual filename, to avoid race conditions when
+        # updating this file.
+        tfile = Filename.temporary(self.getPackageDir().cStr(), '.xml')
+        if doc.SaveFile(tfile.toOsSpecific()):
+            tfile.renameTo(filename)
+        
+    def getUsage(self):
+        """ Returns the xusage element that is read from the usage.xml
+        file, or None if there is no usage.xml file. """
+
+        if not hasattr(PandaModules, 'TiXmlDocument'):
+            return None
+
+        filename = Filename(self.getPackageDir(), self.UsageBasename)
+        doc = TiXmlDocument(filename.toOsSpecific())
+        if not doc.LoadFile():
+            return None
+            
+        xusage = doc.FirstChildElement('usage')
+        if not xusage:
+            return None
+
+        return copy.copy(xusage)
+    

direct/src/p3d/ScanDirectoryNode.py

@@ -0,0 +1,90 @@
+from pandac.PandaModules import VirtualFileSystem, VirtualFileMountSystem, Filename, TiXmlDocument
+vfs = VirtualFileSystem.getGlobalPtr()
+
+class ScanDirectoryNode:
+    """ This class is used to scan a list of files on disk. """
+    
+    def __init__(self, pathname, ignoreUsageXml = False):
+        self.pathname = pathname
+        self.filenames = []
+        self.fileSize = 0
+        self.nested = []
+        self.nestedSize = 0
+
+        xusage = None
+        if not ignoreUsageXml:
+            # Look for a usage.xml file in this directory.  If we find
+            # one, we read it for the file size and then stop here, as
+            # an optimization.
+            usageFilename = Filename(pathname, 'usage.xml')
+            doc = TiXmlDocument(usageFilename.toOsSpecific())
+            if doc.LoadFile():
+                xusage = doc.FirstChildElement('usage')
+                if xusage:
+                    diskSpace = xusage.Attribute('disk_space')
+                    try:
+                        diskSpace = int(diskSpace or '')
+                    except ValueError:
+                        diskSpace = None
+                    if diskSpace is not None:
+                        self.fileSize = diskSpace
+                        return
+
+        for vfile in vfs.scanDirectory(self.pathname):
+            if hasattr(vfile, 'getMount'):
+                if not isinstance(vfile.getMount(), VirtualFileMountSystem):
+                    # Not a real file; ignore it.
+                    continue
+                
+            if vfile.isDirectory():
+                # A nested directory.
+                subdir = ScanDirectoryNode(vfile.getFilename(), ignoreUsageXml = ignoreUsageXml)
+                self.nested.append(subdir)
+                self.nestedSize += subdir.getTotalSize()
+
+            elif vfile.isRegularFile():
+                # A nested file.
+                self.filenames.append(vfile.getFilename())
+                self.fileSize += vfile.getFileSize()
+                
+            else:
+                # Some other wacky file thing.
+                self.filenames.append(vfile.getFilename())
+
+        if xusage:
+            # Now update the usage.xml file with the newly-determined
+            # disk space.
+            xusage.SetAttribute('disk_space', str(self.getTotalSize()))
+            tfile = Filename.temporary(pathname.cStr(), '.xml')
+            if doc.SaveFile(tfile.toOsSpecific()):
+                tfile.renameTo(usageFilename)
+
+    def getTotalSize(self):
+        return self.nestedSize + self.fileSize
+
+    def extractSubdir(self, pathname):
+        """ Finds the ScanDirectoryNode within this node that
+        corresponds to the indicated full pathname.  If it is found,
+        removes it from its parent, and returns it.  If it is not
+        found, returns None. """
+
+        # We could be a little smarter here, but why bother.  Just
+        # recursively search all children.
+        for subdir in self.nested:
+            if subdir.pathname == pathname:
+                self.nested.remove(subdir)
+                self.nestedSize -= subdir.getTotalSize()
+                return subdir
+            
+            result = subdir.extractSubdir(pathname)
+            if result:
+                self.nestedSize -= result.getTotalSize()
+                if subdir.getTotalSize() == 0:
+                    # No other files in the subdirectory that contains
+                    # this package; remove it too.
+                    self.nested.remove(subdir)
+                return result
+
+        return None
+    
+

direct/src/plugin/find_root_dir.cxx

@@ -13,6 +13,8 @@
 ////////////////////////////////////////////////////////////////////
 
 #include "find_root_dir.h"
+#include "mkdir_complete.h"
+#include "get_tinyxml.h"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -35,27 +37,6 @@ DEFINE_KNOWN_FOLDER(FOLDERID_InternetCache, 0x352481E8, 0x33BE, 0x4251, 0xBA, 0x
 #endif  // _WIN32
 
 
-#ifdef _WIN32
-////////////////////////////////////////////////////////////////////
-//     Function: check_root_dir
-//  Description: Tests the proposed root dir string for validity.
-//               Returns true if Panda3D can be successfully created
-//               within the dir, false otherwise.
-////////////////////////////////////////////////////////////////////
-static bool
-check_root_dir(const string &root) {
-  // Attempt to make it first, if possible.
-  CreateDirectory(root.c_str(), NULL);
-  
-  bool isdir = false;
-  DWORD results = GetFileAttributes(root.c_str());
-  if (results != -1) {
-    isdir = (results & FILE_ATTRIBUTE_DIRECTORY) != 0;
-  }
-  return isdir;
-}
-#endif // _WIN32
-
 #ifdef _WIN32
 ////////////////////////////////////////////////////////////////////
 //     Function: get_csidl_dir
@@ -71,7 +52,7 @@ get_csidl_dir(int csidl) {
     string root = buffer;
     root += string("/Panda3D");
     
-    if (check_root_dir(root)) {
+    if (mkdir_complete(root, cerr)) {
       return root;
     }
   }
@@ -109,12 +90,12 @@ wstr_to_string(string &result, const LPWSTR wstr) {
 
 
 ////////////////////////////////////////////////////////////////////
-//     Function: find_root_dir
-//  Description: Returns the path to the installable Panda3D directory
-//               on the user's machine.
+//     Function: find_root_dir_default
+//  Description: Returns the path to the system-default for the root
+//               directory.  This is where we look first.
 ////////////////////////////////////////////////////////////////////
-string
-find_root_dir() {
+static string
+find_root_dir_default() {
 #ifdef _WIN32
   // First, use IEIsProtectedModeProcess() to determine if we are
   // running in IE's "protected mode" under Vista.
@@ -165,7 +146,7 @@ find_root_dir() {
               CoTaskMemFree(cache_path);
               
               root += string("/Panda3D");
-              if (check_root_dir(root)) {
+              if (mkdir_complete(root, cerr)) {
                 FreeLibrary(shell32);
                 FreeLibrary(ieframe);
                 return root;
@@ -199,7 +180,7 @@ find_root_dir() {
           } else {
             CoTaskMemFree(cache_path);
             root += string("/Panda3D");
-            if (check_root_dir(root)) {
+            if (mkdir_complete(root, cerr)) {
               FreeLibrary(ieframe);
               return root;
             }
@@ -236,7 +217,7 @@ find_root_dir() {
   if (GetTempPath(buffer_size, buffer) != 0) {
     root = buffer;
     root += string("Panda3D");
-    if (check_root_dir(root)) {
+    if (mkdir_complete(root, cerr)) {
       return root;
     }
   }
@@ -273,3 +254,42 @@ find_root_dir() {
   return ".";
 }
 
+
+////////////////////////////////////////////////////////////////////
+//     Function: find_root_dir
+//  Description: Returns the path to the installable Panda3D directory
+//               on the user's machine.
+////////////////////////////////////////////////////////////////////
+string
+find_root_dir() {
+  string root = find_root_dir_default();
+
+  // Now look for a config.xml file in that directory, which might
+  // redirect us elsewhere.
+  string config_filename = root + "/config.xml";
+  TiXmlDocument doc(config_filename);
+  if (!doc.LoadFile()) {
+    // No config.xml found, or not valid xml.
+    return root;
+  }
+
+  TiXmlElement *xconfig = doc.FirstChildElement("config");
+  if (xconfig == NULL) {
+    // No <config> element within config.xml.
+    return root;
+  }
+
+  const char *new_root = xconfig->Attribute("root_dir");
+  if (new_root == NULL || *new_root == '\0') {
+    // No root_dir specified.
+    return root;
+  }
+
+  if (!mkdir_complete(new_root, cerr)) {
+    // The specified root_dir wasn't valid.
+    return root;
+  }
+
+  // We've been redirected to another location.  Respect that.
+  return new_root;
+}

direct/src/plugin/p3dHost.cxx

@@ -394,6 +394,26 @@ get_package_desc_file(FileSpec &desc_file,              // out
   return false;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DHost::forget_package
+//       Access: Public
+//  Description: Removes the indicated package from the cache of
+//               packages known by this host.  This is invoked from
+//               the Python side by AppRunner.deletePackages(), so
+//               that we remove the package before deleting its files.
+////////////////////////////////////////////////////////////////////
+void P3DHost::
+forget_package(P3DPackage *package, const string &alt_host) {
+  string key = package->get_package_name() + "_" + package->get_package_version();
+
+  PackageMap &package_map = _packages[alt_host];
+
+  // Hmm, this is a memory leak.  But we allow it to remain, since
+  // it's an unusual circumstance (uninstalling), and it's safer to
+  // leak than to risk a floating pointer.
+  package_map.erase(key);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DHost::migrate_package
 //       Access: Public
@@ -494,6 +514,7 @@ uninstall() {
 
   P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr();
   inst_mgr->delete_directory_recursively(_host_dir);
+  inst_mgr->forget_host(this);
 }
 
 ////////////////////////////////////////////////////////////////////

direct/src/plugin/p3dHost.h

@@ -59,6 +59,7 @@ public:
                              const string &package_name,
                              const string &package_version);
 
+  void forget_package(P3DPackage *package, const string &alt_host = "");
   void migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host);
 
   void choose_random_mirrors(vector<string> &result, int num_mirrors);

direct/src/plugin/p3dInstance.cxx

@@ -1473,8 +1473,8 @@ void P3DInstance::
 uninstall_host() {
   uninstall_packages();
 
+  // Collect the set of hosts referenced by this instance.
   set<P3DHost *> hosts;
-
   Packages::const_iterator pi;
   for (pi = _packages.begin(); pi != _packages.end(); ++pi) {
     P3DPackage *package = (*pi);
@@ -1484,6 +1484,7 @@ uninstall_host() {
     }
   }
 
+  // Uninstall all of them.
   set<P3DHost *>::iterator hi;
   for (hi = hosts.begin(); hi != hosts.end(); ++hi) {
     P3DHost *host = (*hi);
@@ -3098,6 +3099,7 @@ report_package_done(P3DPackage *package, bool success) {
       nout << "No <config> entry in image package\n";
       return;
     }
+    package->mark_used();
 
     for (int i = 0; i < (int)IT_none; ++i) {
       if (_image_files[i]._use_standard_image) {
@@ -3129,6 +3131,8 @@ report_package_done(P3DPackage *package, bool success) {
     // failing to download it) means we can finish checking the
     // authenticity of the p3d file.
 
+    package->mark_used();
+
     // Take down the download progress.
     if (_splash_window != NULL) {
       _splash_window->set_install_progress(0.0, true, 0);
@@ -3145,6 +3149,8 @@ report_package_done(P3DPackage *package, bool success) {
     // Another special case: successfully downloading p3dcert means we
     // can enable the auth button.
 
+    package->mark_used();
+
     // Take down the download progress.
     if (_splash_window != NULL) {
       _splash_window->set_install_progress(0.0, true, 0);

direct/src/plugin/p3dInstanceManager.cxx

@@ -614,6 +614,21 @@ get_host(const string &host_url) {
   return host;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DInstanceManager::forget_host
+//       Access: Public
+//  Description: Removes the indicated host from the cache.
+////////////////////////////////////////////////////////////////////
+void P3DInstanceManager::
+forget_host(P3DHost *host) {
+  const string &host_url = host->get_host_url();
+  
+  // Hmm, this is a memory leak.  But we allow it to remain, since
+  // it's an unusual circumstance (uninstalling), and it's safer to
+  // leak than to risk a floating pointer.
+  _hosts.erase(host_url);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DInstanceManager::get_unique_id
 //       Access: Public
@@ -1395,8 +1410,9 @@ nt_thread_run() {
         P3DInstance *inst = (*ni);
         assert(inst != NULL);
         P3D_request_ready_func *func = inst->get_request_ready_func();
-        assert(func != NULL);
-        (*func)(inst);
+        if (func != NULL) {
+          (*func)(inst);
+        }
       }
       _notify_ready.acquire();
     }

direct/src/plugin/p3dInstanceManager.h

@@ -108,6 +108,7 @@ public:
   void wait_request(double timeout);
 
   P3DHost *get_host(const string &host_url);
+  void forget_host(P3DHost *host);
 
   inline int get_num_instances() const;
 

direct/src/plugin/p3dPackage.cxx

@@ -72,6 +72,7 @@ P3DPackage(P3DHost *host, const string &package_name,
   _failed = false;
   _active_download = NULL;
   _saved_download = NULL;
+  _updated = false;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -210,6 +211,70 @@ remove_instance(P3DInstance *inst) {
   begin_info_download();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: P3DPackage::mark_used
+//       Access: Public
+//  Description: Marks this package as having been "used", for
+//               accounting purposes.
+////////////////////////////////////////////////////////////////////
+void P3DPackage::
+mark_used() {
+  // Unlike the Python variant of this function, we don't mess around
+  // with updating the disk space or anything.
+  string filename = get_package_dir() + "/usage.xml";
+  TiXmlDocument doc(filename);
+  if (!doc.LoadFile()) {
+    TiXmlDeclaration *decl = new TiXmlDeclaration("1.0", "utf-8", "");
+    doc.LinkEndChild(decl);
+  }
+
+  TiXmlElement *xusage = doc.FirstChildElement("usage");
+  if (xusage == NULL) {
+    xusage = new TiXmlElement("usage");
+    doc.LinkEndChild(xusage);
+  }
+
+  time_t now = time(NULL);
+  int count = 0;
+  xusage->Attribute("count_runtime", &count);
+  if (count == 0) {
+    xusage->SetAttribute("first_use", now);
+  }
+
+  ++count;
+  xusage->SetAttribute("count_runtime", count);
+  xusage->SetAttribute("last_use", now);
+
+  if (_updated) {
+    // If we've updated the package, we're no longer sure what its
+    // disk space is.  Remove that from the XML file, so that the
+    // Python code can recompute it later.
+    xusage->RemoveAttribute("disk_space");
+    xusage->SetAttribute("last_update", now);
+  }
+
+  // Write the file to a temporary filename, then atomically move it
+  // to its actual filename, to avoid race conditions.
+  ostringstream strm;
+  strm << get_package_dir() << "/usage_";
+#ifdef _WIN32
+  strm << GetCurrentProcessId();
+#else
+  strm << getpid();
+#endif
+  strm << ".xml";
+  string tfile = strm.str();
+
+  unlink(tfile.c_str());
+  if (doc.SaveFile(tfile)) {
+    if (rename(tfile.c_str(), filename.c_str()) != 0) {
+      // If rename failed, remove the original file first.
+      unlink(filename.c_str());
+      rename(tfile.c_str(), filename.c_str());
+    }
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: P3DPackage::uninstall
 //       Access: Public
@@ -247,6 +312,9 @@ uninstall() {
   _ready = false;
   _failed = false;
   _allow_data_download = false;
+
+  // Make sure the host forgets us too.
+  _host->forget_package(this);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -707,6 +775,7 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) {
 
   inst_mgr->remove_file_from_list(contents, _desc_file_basename);
   inst_mgr->remove_file_from_list(contents, _uncompressed_archive.get_filename());
+  inst_mgr->remove_file_from_list(contents, "usage.xml");
   Extracts::iterator ei;
   for (ei = _extracts.begin(); ei != _extracts.end(); ++ei) {
     inst_mgr->remove_file_from_list(contents, (*ei).get_filename());
@@ -727,6 +796,7 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) {
     chmod(pathname.c_str(), 0644);
 #endif
     unlink(pathname.c_str());
+    _updated = true;
   }
 
   // Verify the uncompressed archive.
@@ -1509,6 +1579,7 @@ do_step(bool download_finished) {
 
   if (_download->get_download_success()) {
     // The Download object has already validated the hash.
+    _package->_updated = true;
     return IT_step_complete;
 
   } else if (_download->get_download_terminated()) {
@@ -1801,6 +1872,7 @@ thread_step() {
   unlink(source_pathname.c_str());
 
   // All done uncompressing.
+  _package->_updated = true;
   return IT_step_complete;
 }
 
@@ -1847,6 +1919,7 @@ thread_step() {
     return IT_step_failed;
   }
 
+  _package->_updated = true;
   return IT_step_complete;
 }
 
@@ -1901,6 +1974,7 @@ thread_step() {
     return IT_step_failed;
   }
 
+  _package->_updated = true;
   return IT_step_complete;
 }
 

direct/src/plugin/p3dPackage.h

@@ -69,6 +69,7 @@ public:
   void add_instance(P3DInstance *inst);
   void remove_instance(P3DInstance *inst);
 
+  void mark_used();
   void uninstall();
 
   TiXmlElement *make_xml();
@@ -290,6 +291,7 @@ private:
 
   size_t _unpack_size;
   Extracts _extracts;
+  bool _updated;
 
   static const double _download_factor;
   static const double _uncompress_factor;

direct/src/plugin/p3dSession.cxx

@@ -958,6 +958,8 @@ start_p3dpython(P3DInstance *inst) {
     _env += string("_ROOT=");
     _env += package->get_package_dir();
     _env += '\0';
+
+    package->mark_used();
   }
 
   // Check for a few tokens that have special meaning at this level.