patching

direct/src/p3d/AppRunner.py

@@ -25,7 +25,10 @@ if 'VFSImporter' in sys.modules:
     direct.showbase.VFSImporter = VFSImporter
     sys.modules['direct.showbase.VFSImporter'] = VFSImporter
 else:
-    # Otherwise, we can import the VFSImporter normally.
+    # Otherwise, we can import the VFSImporter normally.  We have to
+    # import PandaModules first, to get the funny renaming with
+    # pandaexpress.
+    from pandac import PandaModules
     from direct.showbase import VFSImporter
 
 from direct.showbase.DirectObject import DirectObject

direct/src/p3d/DWBPackageInstaller.py

@@ -4,7 +4,16 @@ from direct.gui import DirectGuiGlobals as DGG
 
 class DWBPackageInstaller(DirectWaitBar, PackageInstaller):
     """ This class presents a PackageInstaller that also inherits from
-    DirectWaitBar, so it updates its own GUI as it downloads. """
+    DirectWaitBar, so it updates its own GUI as it downloads.
+
+    Specify perPackage = True to make the progress bar reset for each
+    package, or False (the default) to show one continuous progress
+    bar for all packages.
+
+    Specify updateText = True (the default) to update the text label
+    with the name of the package or False to leave it up to you to set
+    it.
+    """
 
     def __init__(self, appRunner, parent = None, **kw):
         PackageInstaller.__init__(self, appRunner)
@@ -18,7 +27,9 @@ class DWBPackageInstaller(DirectWaitBar, PackageInstaller):
             ('barRelief',      DGG.RAISED,         self.setBarRelief),
             ('text',           'Starting',         self.setText),
             ('text_pos',       (0, -0.025),        None),
-            ('text_scale',     0.1,                None)
+            ('text_scale',     0.1,                None),
+            ('perPackage',     False,              None),
+            ('updateText',     True,               None),
             )
         self.defineoptions(kw, optiondefs)
         DirectWaitBar.__init__(self, parent, **kw)
@@ -40,8 +51,18 @@ class DWBPackageInstaller(DirectWaitBar, PackageInstaller):
         """ This callback is made for each package between
         downloadStarted() and downloadFinished() to indicate the start
         of a new package. """
-        self['text'] = 'Installing %s' % (package.displayName)
+        if self['updateText']:
+            self['text'] = 'Installing %s' % (package.displayName)
         self.show()
+
+    def packageProgress(self, package, progress):
+        """ This callback is made repeatedly between packageStarted()
+        and packageFinished() to update the current progress on the
+        indicated package only.  The progress value ranges from 0
+        (beginning) to 1 (complete). """
+
+        if self['perPackage']:
+            self['value'] = progress * self['range']
         
     def downloadProgress(self, overallProgress):
         """ This callback is made repeatedly between downloadStarted()
@@ -49,7 +70,8 @@ class DWBPackageInstaller(DirectWaitBar, PackageInstaller):
         all packages.  The progress value ranges from 0 (beginning) to
         1 (complete). """
 
-        self['value'] = overallProgress * self['range']
+        if not self['perPackage']:
+            self['value'] = overallProgress * self['range']
 
     def downloadFinished(self, success):
         """ This callback is made when all of the packages have been

direct/src/p3d/FileSpec.py

@@ -8,10 +8,12 @@ class FileSpec:
     the xml. """
 
     def __init__(self):
-        pass
+        self.actualFile = None
 
-    def fromFile(self, packageDir, filename):
-        """ Reads the file information from the indicated file. """
+    def fromFile(self, packageDir, filename, st = None):
+        """ Reads the file information from the indicated file.  If st
+        is supplied, it is the result of os.stat on the filename. """
+        
         vfs = VirtualFileSystem.getGlobalPtr()
 
         filename = Filename(filename)
@@ -20,7 +22,8 @@ class FileSpec:
         self.filename = filename.cStr()
         self.basename = filename.getBasename()
 
-        st = os.stat(pathname.toOsSpecific())
+        if st is None:
+            st = os.stat(pathname.toOsSpecific())
         self.size = st.st_size
         self.timestamp = st.st_mtime
 
@@ -33,17 +36,20 @@ class FileSpec:
         element. """
         
         self.filename = xelement.Attribute('filename')
-        self.basename = Filename(self.filename).getBasename()
+        self.basename = None
+        if self.filename:
+            self.basename = Filename(self.filename).getBasename()
+            
         size = xelement.Attribute('size')
         try:
             self.size = int(size)
-        except ValueError:
+        except:
             self.size = 0
 
         timestamp = xelement.Attribute('timestamp')
         try:
             self.timestamp = int(timestamp)
-        except ValueError:
+        except:
             self.timestamp = 0
 
         self.hash = xelement.Attribute('hash')
@@ -52,10 +58,23 @@ class FileSpec:
         """ Adds the file information to the indicated XML
         element. """
 
-        xelement.SetAttribute('filename', self.filename)
-        xelement.SetAttribute('size', str(self.size))
-        xelement.SetAttribute('timestamp', str(self.timestamp))
-        xelement.SetAttribute('hash', self.hash)
+        if self.filename:
+            xelement.SetAttribute('filename', self.filename)
+        if self.size:
+            xelement.SetAttribute('size', str(self.size))
+        if self.timestamp:
+            xelement.SetAttribute('timestamp', str(self.timestamp))
+        if self.hash:
+            xelement.SetAttribute('hash', self.hash)
+
+    def storeMiniXml(self, xelement):
+        """ Adds the just the "mini" file information--size and
+        hash--to the indicated XML element. """
+
+        if self.size:
+            xelement.SetAttribute('size', str(self.size))
+        if self.hash:
+            xelement.SetAttribute('hash', self.hash)
             
     def quickVerify(self, packageDir = None, pathname = None):
         """ Performs a quick test to ensure the file has not been
@@ -89,7 +108,7 @@ class FileSpec:
 
         # If the size is right but the timestamp is wrong, the file
         # soft-fails.  We follow this up with a hash check.
-        if not self.checkHash(pathname):
+        if not self.checkHash(packageDir, st):
             # Hard fail, the hash is wrong.
             #print "hash check wrong: %s" % (pathname)
             return False
@@ -126,7 +145,7 @@ class FileSpec:
             #print "size wrong: %s" % (pathname)
             return False
 
-        if not self.checkHash(pathname):
+        if not self.checkHash(packageDir, st):
             # Hard fail, the hash is wrong.
             #print "hash check wrong: %s" % (pathname)
             return False
@@ -141,11 +160,14 @@ class FileSpec:
 
         return True
 
-    def checkHash(self, pathname):
+    def checkHash(self, packageDir, st):
         """ Returns true if the file has the expected md5 hash, false
-        otherwise. """
+        otherwise.  As a side effect, stores a FileSpec corresponding
+        to the on-disk file in self.actualFile. """
 
-        hv = HashVal()
-        hv.hashFile(pathname)
-        return (hv.asHex() == self.hash)
+        fileSpec = FileSpec()
+        fileSpec.fromFile(packageDir, self.filename, st = st)
+        self.actualFile = fileSpec
+
+        return (fileSpec.hash == self.hash)
     

direct/src/p3d/PackageInfo.py

@@ -1,4 +1,4 @@
-from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, TiXmlDocument, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath
+from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, TiXmlDocument, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, Patchfile
 from direct.p3d.FileSpec import FileSpec
 from direct.showbase import VFSImporter
 import os
@@ -10,6 +10,36 @@ class PackageInfo:
     can be (or has been) installed into the current runtime.  It is
     the Python equivalent of the P3DPackage class in the core API. """
 
+    # Weight factors for computing download progress.  This
+    # attempts to reflect the relative time-per-byte of each of
+    # these operations.
+    downloadFactor = 1
+    uncompressFactor = 0.01
+    unpackFactor = 0.01
+    patchFactor = 0.01
+
+    class InstallStep:
+        """ This class is one step of the installPlan list; it
+        represents a single atomic piece of the installation step, and
+        the relative effort of that piece.  When the plan is executed,
+        it will call the saved function pointer here. """
+        def __init__(self, func, bytes, factor):
+            self.func = func
+            self.bytesNeeded = bytes
+            self.bytesDone = 0
+            self.bytesFactor = factor
+
+        def getEffort(self):
+            """ Returns the relative amount of effort of this step. """
+            return self.bytesNeeded * self.bytesFactor
+
+        def getProgress(self):
+            """ Returns the progress of this step, in the range
+            0..1. """
+            if self.bytesNeeded == 0:
+                return 1
+            return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
+    
     def __init__(self, host, packageName, packageVersion, platform = None):
         self.host = host
         self.packageName = packageName
@@ -32,40 +62,28 @@ class PackageInfo:
         self.uncompressedArchive = None
         self.compressedArchive = None
         self.extracts = []
-
-        # These are incremented during downloadPackage().
-        self.bytesDownloaded = 0
-        self.bytesUncompressed = 0
-        self.bytesUnpacked = 0
+        self.installPlans = None
+ 
+        # This is updated during downloadPackage().  It is in the
+        # range 0..1.
+        self.downloadProgress = 0
         
         # This is set true when the package file has been fully
         # downloaded and unpackaged.
         self.hasPackage = False
 
-    def getDownloadSize(self):
-        """ Returns the number of bytes we will need to download in
-        order to install this package. """
-        if self.hasPackage:
-            return 0
-        return self.compressedArchive.size
+    def getDownloadEffort(self):
+        """ Returns the relative amount of effort it will take to
+        download this package.  The units are meaningless, except
+        relative to other packges."""
 
-    def getUncompressSize(self):
-        """ Returns the number of bytes we will need to uncompress in
-        order to install this package. """
-        if self.hasPackage:
+        if not self.installPlans:
             return 0
-        return self.uncompressedArchive.size
-
-    def getUnpackSize(self):
-        """ Returns the number of bytes that we will need to unpack
-        when installing the package. """
 
-        if self.hasPackage:
-            return 0
-
-        size = 0
-        for file in self.extracts:
-            size += file.size
+        # Return the size of plan A, assuming it will work.
+        plan = self.installPlans[0]
+        size = sum(map(lambda step: step.getEffort(), plan))
+        
         return size
 
     def setupFilenames(self):
@@ -189,6 +207,71 @@ class PackageInfo:
         # We need to download an update.
         self.hasPackage = False
 
+        # Now determine what we will need to download, and build a
+        # plan (or two) to download it all.
+        self.installPlans = None
+
+        # We know we will at least need to unpackage the archive at
+        # the end.
+        unpackSize = 0
+        for file in self.extracts:
+            unpackSize += file.size
+        step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor)
+        planA = [step]
+
+        # If the uncompressed archive file is good, that's all we'll
+        # need to do.
+        self.uncompressedArchive.actualFile = None
+        if self.uncompressedArchive.quickVerify(self.packageDir):
+            self.installPlans = [planA]
+            return
+
+        # Maybe the compressed archive file is good.
+        if self.compressedArchive.quickVerify(self.packageDir):
+            uncompressSize = self.uncompressedArchive.size
+            step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
+            planA = [step] + planA
+            self.installPlans = [planA]
+            return
+
+        # Maybe we can download one or more patches.  We'll come back
+        # to that in a minute as plan A.  For now, construct on plan
+        # B, which will be to download the whole archive.
+        planB = planA[:]
+
+        uncompressSize = self.uncompressedArchive.size
+        step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
+        planB = [step] + planB
+
+        downloadSize = self.compressedArchive.size
+        func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec)
+
+        step = self.InstallStep(func, downloadSize, self.downloadFactor)
+        planB = [step] + planB
+
+        # Now look for patches.  Start with the md5 hash from the
+        # uncompressedArchive file we have on disk, and see if we can
+        # find a patch chain from this file to our target.
+        pathname = Filename(self.packageDir, self.uncompressedArchive.filename)
+        fileSpec = self.uncompressedArchive.actualFile
+        if fileSpec is None and pathname.exists():
+            fileSpec = FileSpec()
+            fileSpec.fromFile(self.packageDir, self.uncompressedArchive.filename)
+        plan = None
+        if fileSpec:
+            plan = self.__findPatchChain(fileSpec)
+        if plan:
+            # We can download patches.  Great!  That means this is
+            # plan A, and the full download is plan B (in case
+            # something goes wrong with the patching).
+            planA = plan + planA
+            self.installPlans = [planA, planB]
+        else:
+            # There are no patches to download, oh well.  Stick with
+            # plan B as the only plan.
+            self.installPlans = [planB]
+        
+
     def __checkArchiveStatus(self):
         """ Returns true if the archive and all extractable files are
         already correct on disk, false otherwise. """
@@ -207,6 +290,15 @@ class PackageInfo:
 
         return allExtractsOk
 
+    def __updateStepProgress(self, step):
+        """ This callback is made from within the several step
+        functions as the download step proceeds.  It updates
+        self.downloadProgress with the current progress, so the caller
+        can asynchronously query this value. """
+
+        size = self.totalPlanCompleted + self.currentStepEffort * step.getProgress()
+        self.downloadProgress = min(float(size) / float(self.totalPlanSize), 1)
+    
     def downloadPackage(self, http):
         """ Downloads the package file, synchronously, then
         uncompresses and unpacks it.  Returns true on success, false
@@ -218,68 +310,165 @@ class PackageInfo:
             # We've already got one.
             return True
 
-        if self.uncompressedArchive.quickVerify(self.packageDir):
-            return self.__unpackArchive()
+        # We should have an install plan by the time we get here.
+        assert self.installPlans
 
-        if self.compressedArchive.quickVerify(self.packageDir):
-            return self.__uncompressArchive()
+        self.http = http
+        for plan in self.installPlans:
+            self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan))
+            self.totalPlanCompleted = 0
+            self.downloadProgress = 0
+
+            planFailed = False
+            for step in plan:
+                self.currentStepEffort = step.getEffort()
 
+                if not step.func(step):
+                    planFailed = True
+                    break
+                self.totalPlanCompleted += self.currentStepEffort
+                
+            if not planFailed:
+                # Successfully downloaded!
+                return True
+
+        # All plans failed.
+        return False
+
+    def __findPatchChain(self, fileSpec):
+        """ Finds the chain of patches that leads from the indicated
+        patch version to the current patch version.  If found,
+        constructs an installPlan that represents the steps of the
+        patch installation; otherwise, returns None. """
+
+        from direct.p3d.PatchMaker import PatchMaker
+
+        patchMaker = PatchMaker(self.packageDir)
+        package = patchMaker.readPackageDescFile(self.descFileBasename)
+        patchMaker.buildPatchChains()
+        fromPv = patchMaker.getPackageVersion(package.getGenericKey(fileSpec))
+        toPv = package.currentPv
+        patchChain = toPv.getPatchChain(fromPv)
+
+        if patchChain is None:
+            # No path.
+            patchMaker.cleanup()
+            return None
+
+        plan = []
+        for patchfile in patchChain:
+            downloadSize = patchfile.file.size
+            func = lambda step, fileSpec = patchfile.file: self.__downloadFile(step, fileSpec)
+            step = self.InstallStep(func, downloadSize, self.downloadFactor)
+            plan.append(step)
+
+            patchSize = patchfile.targetFile.size
+            func = lambda step, patchfile = patchfile: self.__applyPatch(step, patchfile)
+            step = self.InstallStep(func, patchSize, self.patchFactor)
+            plan.append(step)
+
+        patchMaker.cleanup()
+        return plan
+
+    def __downloadFile(self, step, fileSpec):
+        """ Downloads the indicated file from the host into
+        packageDir.  Returns true on success, false on failure. """
+        
         url = self.descFileUrl.rsplit('/', 1)[0]
-        url += '/' + self.compressedArchive.filename
+        url += '/' + fileSpec.filename
         url = DocumentSpec(url)
         print "Downloading %s" % (url)
 
-        targetPathname = Filename(self.packageDir, self.compressedArchive.filename)
+        targetPathname = Filename(self.packageDir, fileSpec.filename)
         targetPathname.setBinary()
         
-        channel = http.makeChannel(False)
+        channel = self.http.makeChannel(False)
         channel.beginGetDocument(url)
         channel.downloadToFile(targetPathname)
         while channel.run():
-            self.bytesDownloaded = channel.getBytesDownloaded()
+            step.bytesDone = channel.getBytesDownloaded()
+            self.__updateStepProgress(step)
             Thread.considerYield()
-        self.bytesDownloaded = channel.getBytesDownloaded()
+        step.bytesDone = channel.getBytesDownloaded()
+        self.__updateStepProgress(step)
         if not channel.isValid():
             print "Failed to download %s" % (url)
             return False
 
-        if not self.compressedArchive.fullVerify(self.packageDir):
-            print "after downloading, %s still incorrect" % (
-                self.compressedArchive.filename)
+        if not fileSpec.fullVerify(self.packageDir):
+            print "after downloading, %s incorrect" % (
+                fileSpec.filename)
             return False
         
-        return self.__uncompressArchive()
+        return True
+
+    def __applyPatch(self, step, patchfile):
+        """ Applies the indicated patching in-place to the current
+        uncompressed archive.  The patchfile is removed after the
+        operation.  Returns true on success, false on failure. """
+
+        origPathname = Filename(self.packageDir, self.uncompressedArchive.filename)
+        patchPathname = Filename(self.packageDir, patchfile.file.filename)
+        result = Filename.temporary('', 'patch_')
+        print "Patching %s with %s" % (origPathname, patchPathname)
+
+        p = Patchfile()  # The C++ class
 
-    def __uncompressArchive(self):
+        ret = p.initiate(patchPathname, origPathname, result)
+        if ret == EUSuccess:
+            ret = p.run()
+        while ret == EUOk:
+            step.bytesDone = step.bytesNeeded * p.getProgress()
+            self.__updateStepProgress(step)
+            Thread.considerYield()
+            ret = p.run()
+        del p
+        patchPathname.unlink()
+        
+        if ret < 0:
+            print "Patching failed."
+            result.unlink()
+            return False
+
+        if not result.renameTo(origPathname):
+            print "Couldn't rename %s to %s" % (result, origPathname)
+            return False
+            
+        return True
+
+    def __uncompressArchive(self, step):
         """ Turns the compressed archive into the uncompressed
-        archive, then unpacks it.  Returns true on success, false on
-        failure. """
+        archive.  Returns true on success, false on failure. """
 
         sourcePathname = Filename(self.packageDir, self.compressedArchive.filename)
         targetPathname = Filename(self.packageDir, self.uncompressedArchive.filename)
-
+        print "Uncompressing %s to %s" % (sourcePathname, targetPathname)
         decompressor = Decompressor()
         decompressor.initiate(sourcePathname, targetPathname)
         totalBytes = self.uncompressedArchive.size
         result = decompressor.run()
         while result == EUOk:
-            self.bytesUncompressed = int(totalBytes * decompressor.getProgress())
+            step.bytesDone = int(totalBytes * decompressor.getProgress())
+            self.__updateStepProgress(step)
             result = decompressor.run()
             Thread.considerYield()
 
         if result != EUSuccess:
             return False
             
-        self.bytesUncompressed = totalBytes
+        step.bytesDone = totalBytes
+        self.__updateStepProgress(step)
 
         if not self.uncompressedArchive.quickVerify(self.packageDir):
             print "after uncompressing, %s still incorrect" % (
                 self.uncompressedArchive.filename)
             return False
 
-        return self.__unpackArchive()
+        # Now we can safely remove the compressed archive.
+        sourcePathname.unlink()
+        return True
     
-    def __unpackArchive(self):
+    def __unpackArchive(self, step):
         """ Unpacks any files in the archive that want to be unpacked
         to disk. """
 
@@ -289,13 +478,14 @@ class PackageInfo:
             return True
 
         mfPathname = Filename(self.packageDir, self.uncompressedArchive.filename)
+        print "Unpacking %s" % (mfPathname)
         mf = Multifile()
         if not mf.openRead(mfPathname):
             print "Couldn't open %s" % (mfPathname)
             return False
         
         allExtractsOk = True
-        self.bytesUnpacked = 0
+        step.bytesDone = 0
         for file in self.extracts:
             i = mf.findSubfile(file.filename)
             if i == -1:
@@ -317,7 +507,8 @@ class PackageInfo:
             # Make sure it's executable.
             os.chmod(targetPathname.toOsSpecific(), 0755)
 
-            self.bytesUnpacked += file.size
+            step.bytesDone += file.size
+            self.__updateStepProgress(step)
             Thread.considerYield()
 
         if not allExtractsOk:

direct/src/p3d/PackageInstaller.py

@@ -35,13 +35,6 @@ class PackageInstaller(DirectObject):
     class PendingPackage:
         """ This class describes a package added to the installer for
         download. """
-
-        # Weight factors for computing download progress.  This
-        # attempts to reflect the relative time-per-byte of each of
-        # these operations.
-        downloadFactor = 1
-        uncompressFactor = 0.02
-        unpackFactor = 0.01
         
         def __init__(self, packageName, version, host):
             self.packageName = packageName
@@ -63,28 +56,15 @@ class PackageInstaller(DirectObject):
             # bytes uncompressed, and bytes extracted; and each of
             # which is weighted differently into one grand total.  So,
             # the total doesn't really represent bytes; it's a
-            # unitless number, which means something only as a ratio.
-            self.targetDownloadSize = 0
-
-        def getCurrentDownloadSize(self):
-            """ Returns the current amount of stuff we have processed
-            so far in the download. """
-            if self.done:
-                return self.targetDownloadSize
-
-            return (
-                self.package.bytesDownloaded * self.downloadFactor +
-                self.package.bytesUncompressed * self.uncompressFactor +
-                self.package.bytesUnpacked * self.unpackFactor)
+            # unitless number, which means something only as a ratio
+            # to other packages.
+            self.downloadEffort = 0
 
         def getProgress(self):
             """ Returns the download progress of this package in the
             range 0..1. """
 
-            if not self.targetDownloadSize:
-                return 1
-
-            return float(self.getCurrentDownloadSize()) / float(self.targetDownloadSize)
+            return self.package.downloadProgress
 
         def getDescFile(self, http):
             """ Synchronously downloads the desc files required for
@@ -104,11 +84,8 @@ class PackageInstaller(DirectObject):
                 return False
 
             self.package.checkStatus()
-            self.targetDownloadSize = (
-                self.package.getDownloadSize() * self.downloadFactor +
-                self.package.getUncompressSize() * self.uncompressFactor +
-                self.package.getUnpackSize() * self.unpackFactor)
-            
+            self.downloadEffort = self.package.getDownloadEffort()
+
             return True
 
     def __init__(self, appRunner, taskChain = 'install'):
@@ -484,7 +461,7 @@ class PackageInstaller(DirectObject):
         # Now serve this one package.
         messenger.send('PackageInstaller-%s-packageStarted' % self.uniqueId,
                        [pp], taskChain = 'default')
-        
+
         if not pp.package.downloadPackage(self.appRunner.http):
             self.__donePackage(pp, False)
             return task.cont
@@ -527,18 +504,19 @@ class PackageInstaller(DirectObject):
                 self.progressTask = None
                 return task.done
 
-            targetDownloadSize = 0
+            downloadEffort = 0
             currentDownloadSize = 0
             for pp in self.packages:
-                targetDownloadSize += pp.targetDownloadSize
-                currentDownloadSize += pp.getCurrentDownloadSize()
+                downloadEffort += pp.downloadEffort
+                packageProgress = pp.getProgress()
+                currentDownloadSize += pp.downloadEffort * packageProgress
                 if pp.calledPackageStarted and not pp.calledPackageFinished:
-                    self.packageProgress(pp.package, pp.getProgress())
+                    self.packageProgress(pp.package, packageProgress)
 
-            if not targetDownloadSize:
+            if not downloadEffort:
                 progress = 1
             else:
-                progress = float(currentDownloadSize) / float(targetDownloadSize)
+                progress = float(currentDownloadSize) / float(downloadEffort)
             self.downloadProgress(progress)
             
         finally:

direct/src/p3d/Packager.py

@@ -943,7 +943,7 @@ class Packager:
             it.  We need this to preserve the list of patchfiles
             between sessions. """
 
-            self.patchVersion = '1'
+            self.patchVersion = None
             self.patches = []
             
             packageDescFullpath = Filename(self.packager.installDir, self.packageDesc)
@@ -982,7 +982,8 @@ class Packager:
             if self.version:
                 xpackage.SetAttribute('version', self.version)
 
-            xpackage.SetAttribute('last_patch_version', self.patchVersion)
+            if self.patchVersion:
+                xpackage.SetAttribute('last_patch_version', self.patchVersion)
 
             self.__addConfigs(xpackage)
 
@@ -1638,6 +1639,16 @@ class Packager:
 
         self.writeContentsFile()
 
+    def buildPatches(self, packages):
+        """ Call this after calling close(), to build patches for the
+        indicated packages. """
+
+        packageNames = map(lambda package: package.packageName, packages)
+
+        from PatchMaker import PatchMaker
+        pm = PatchMaker(self.installDir)
+        pm.buildPatches(packageNames = packageNames)
+
     def readPackageDef(self, packageDef):
         """ Reads the named .pdef file and constructs the packages
         indicated within it.  Raises an exception if the pdef file is

direct/src/p3d/PatchMaker.py

@@ -4,7 +4,8 @@ from pandac.PandaModules import *
 class PatchMaker:
     """ This class will operate on an existing package install
     directory, as generated by the Packager, and create patchfiles
-    between versions as needed. """
+    between versions as needed.  It is also used at runtime, to apply
+    the downloaded patches. """
 
     class PackageVersion:
         """ A specific patch version of a package.  This is not just
@@ -12,15 +13,15 @@ class PatchMaker:
         particular patch version, which increments independently of
         the "version". """
         
-        def __init__(self, packageName, platform, version, host, hash):
+        def __init__(self, packageName, platform, version, host, file):
             self.packageName = packageName
             self.platform = platform
             self.version = version
             self.host = host
-            self.hash = hash
+            self.file = file
 
-            # The Package object that produces this version, in the
-            # current form or the base form, respectively.
+            # The Package object that produces this version, if this
+            # is the current form or the base form, respectively.
             self.packageCurrent = None
             self.packageBase = None
 
@@ -38,55 +39,108 @@ class PatchMaker:
             if self.tempFile:
                 self.tempFile.unlink()
 
-        def getFile(self):
-            """ Returns the Filename of the archive file associated
-            with this version.  If the file doesn't actually exist on
-            disk, a temporary file will be created.  Returns None if
-            the file can't be recreated. """
+        def getPatchChain(self, startPv):
+            """ Returns a list of patches that, when applied in
+            sequence to the indicated patchVersion object, will
+            produce this patchVersion object.  Returns None if no
+            chain can be found. """
+
+            if self is startPv:
+                # We're already here.  A zero-length patch chain is
+                # therefore the answer.
+                return []
 
+            bestPatchChain = None
+            for patchfile in self.fromPatches:
+                fromPv = patchfile.fromPv
+                patchChain = fromPv.getPatchChain(startPv)
+                if patchChain is not None:
+                    # There's a path through this patchfile.
+                    patchChain = patchChain + [patchfile]
+                    if bestPatchChain is None or len(patchChain) < len(bestPatchChain):
+                        bestPatchChain = patchChain
+
+            # Return the shortest path found, or None if there were no
+            # paths found.
+            return bestPatchChain
+
+        def getRecreateFilePlan(self):
+            """ Returns the tuple (startFile, plan), describing how to
+            recreate the archive file for this version.  startFile is
+            the Filename of the file to start with, and plan is a list
+            of tuples (patchfile, pv), listing the patches to apply in
+            sequence, and the packageVersion object associated with
+            each patch.  Returns (None, None) if there is no way to
+            recreate this archive file.  """
+            
             if self.tempFile:
-                return self.tempFile
+                return (self.tempFile, [])
 
             if self.packageCurrent:
                 package = self.packageCurrent
-                return Filename(package.packageDir, package.currentFile.filename)
+                return (Filename(package.packageDir, package.currentFile.filename), [])
             if self.packageBase:
                 package = self.packageBase
-                return Filename(package.packageDir, package.baseFile.filename)
+                return (Filename(package.packageDir, package.baseFile.filename), [])
 
             # We'll need to re-create the file.
+            bestPlan = None
+            bestStartFile = None
             for patchfile in self.fromPatches:
                 fromPv = patchfile.fromPv
-                prevFile = fromPv.getFile()
-                if prevFile:
-                    patchFilename = Filename(patchfile.package.packageDir, patchfile.file.filename)
-                    result = self.applyPatch(prevFile, patchFilename, patchfile.toHash)
-                    if result:
-                        self.tempFile = result
-                        return result
-
-            # Couldn't re-create the file.
-            return None
+                startFile, plan = fromPv.getRecreateFilePlan()
+                if plan is not None:
+                    # There's a path through this patchfile.
+                    plan = plan + [(patchfile, self)]
+                    if bestPlan is None or len(plan) < len(bestPlan):
+                        bestPlan = plan
+                        bestStartFile = startFile
+
+            # Return the shortest path found, or None if there were no
+            # paths found.
+            return (bestStartFile, bestPlan)
 
-        def applyPatch(self, origFile, patchFilename, targetHash):
+        def getFile(self):
+            """ Returns the Filename of the archive file associated
+            with this version.  If the file doesn't actually exist on
+            disk, a temporary file will be created.  Returns None if
+            the file can't be recreated. """
+
+            startFile, plan = self.getRecreateFilePlan()
+            if not plan:
+                # If plan is a zero-length list, we're already
+                # here--return startFile.  If plan is None, there's no
+                # solution, and startFile is None.  In either case, we
+                # can return startFile.
+                return startFile
+
+            # If plan is a non-empty list, we have to walk the list to
+            # apply the patch plan.
+            prevFile = startFile
+            for patchfile, pv in plan:
+                fromPv = patchfile.fromPv
+                patchFilename = Filename(patchfile.package.packageDir, patchfile.file.filename)
+                result = self.applyPatch(prevFile, patchFilename)
+                if not result:
+                    # Failure trying to re-create the file.
+                    return None
+                
+                pv.tempFile = result
+                prevFile = result
+
+            # Successfully patched.
+            assert pv is self and prevFile is self.tempFile
+            return prevFile
+
+        def applyPatch(self, origFile, patchFilename):
             """ Applies the named patch to the indicated original
             file, storing the results in a temporary file, and returns
             that temporary Filename.  Returns None on failure. """
 
             result = Filename.temporary('', 'patch_')
-            print "patching %s + %s -> %s" % (
-                origFile, patchFilename, result)
-
             p = Patchfile()
             if not p.apply(patchFilename, origFile, result):
-                print "patching failed."
-                return None
-
-            hv = HashVal()
-            hv.hashFile(result)
-            if hv.asHex() != targetHash:
-                print "patching produced incorrect results."
-                result.unlink()
+                print "Internal patching failed: %s" % (patchFilename)
                 return None
 
             return result
@@ -113,18 +167,17 @@ class PatchMaker:
             self.version = package.version
             self.host = None
 
-        def getFromKey(self):
-            return (self.packageName, self.platform, self.version, self.host, self.fromHash)
+        def getSourceKey(self):
+            return (self.packageName, self.platform, self.version, self.host, self.sourceFile)
 
-        def getToKey(self):
-            return (self.packageName, self.platform, self.version, self.host, self.toHash)
+        def getTargetKey(self):
+            return (self.packageName, self.platform, self.version, self.host, self.targetFile)
 
-        def fromFile(self, packageDir, patchFilename,
-                     fromHash, toHash):
+        def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
             self.file = FileSpec()
             self.file.fromFile(packageDir, patchFilename)
-            self.fromHash = fromHash
-            self.toHash = toHash
+            self.sourceFile = sourceFile
+            self.targetFile = targetFile
 
         def loadXml(self, xpatch):
             self.packageName = xpatch.Attribute('name') or self.packageName
@@ -135,8 +188,15 @@ class PatchMaker:
             self.file = FileSpec()
             self.file.loadXml(xpatch)
 
-            self.fromHash = xpatch.Attribute('from_hash')
-            self.toHash = xpatch.Attribute('to_hash')
+            xsource = xpatch.FirstChildElement('source')
+            if xsource:
+                self.sourceFile = FileSpec()
+                self.sourceFile.loadXml(xsource)
+
+            xtarget = xpatch.FirstChildElement('target')
+            if xtarget:
+                self.targetFile = FileSpec()
+                self.targetFile.loadXml(xtarget)
 
         def makeXml(self, package):
             xpatch = TiXmlElement('patch')
@@ -152,8 +212,13 @@ class PatchMaker:
 
             self.file.storeXml(xpatch)
 
-            xpatch.SetAttribute('from_hash', self.fromHash)
-            xpatch.SetAttribute('to_hash', self.toHash)
+            xsource = TiXmlElement('source')
+            self.sourceFile.storeMiniXml(xsource)
+            xpatch.InsertEndChild(xsource)
+
+            xtarget = TiXmlElement('target')
+            self.targetFile.storeMiniXml(xtarget)
+            xpatch.InsertEndChild(xtarget)
 
             return xpatch
 
@@ -166,16 +231,22 @@ class PatchMaker:
             self.packageDesc = packageDesc
             self.patchMaker = patchMaker
             self.patchVersion = 1
+            self.currentPv = None
+            self.basePv = None
 
             self.doc = None
             self.anyChanges = False
             self.patches = []
 
         def getCurrentKey(self):
-            return (self.packageName, self.platform, self.version, self.host, self.currentFile.hash)
+            return (self.packageName, self.platform, self.version, self.host, self.currentFile)
 
         def getBaseKey(self):
-            return (self.packageName, self.platform, self.version, self.host, self.baseFile.hash)
+            return (self.packageName, self.platform, self.version, self.host, self.baseFile)
+
+        def getGenericKey(self, fileSpec):
+            """ Returns the key that has the indicated FileSpec. """
+            return (self.packageName, self.platform, self.version, self.host, fileSpec)
 
         def readDescFile(self):
             """ Reads the existing package.xml file and stores
@@ -281,20 +352,44 @@ class PatchMaker:
     def __init__(self, installDir):
         self.installDir = installDir
         self.packageVersions = {}
+        self.packages = []
 
-    def run(self):
+    def buildPatches(self, packageNames = None):
+        """ Makes the patches required in a particular directory
+        structure on disk.  If packageNames is None, this makes
+        patches for all packages; otherwise, it should be a list of
+        package name strings, limiting the set of packages that are
+        processed. """
+        
         if not self.readContentsFile():
             return False
         self.buildPatchChains()
-        self.processPackages()
+        if packageNames is None:
+            self.processAllPackages()
+        else:
+            self.processSomePackages(packageNames)
 
         self.cleanup()
         return True
 
     def cleanup(self):
+        """ Should be called on exit to remove temporary files and
+        such created during processing. """
+        
         for pv in self.packageVersions.values():
             pv.cleanup()
 
+    def readPackageDescFile(self, descFilename):
+        """ Reads a desc file associated with a particular package,
+        and adds the package to self.packageVersions.  Returns the
+        Package object. """
+
+        package = self.Package(Filename(descFilename), self)
+        package.readDescFile()
+        self.packages.append(package)
+
+        return package
+
     def readContentsFile(self):
         """ Reads the contents.xml file at the beginning of
         processing. """
@@ -306,7 +401,6 @@ class PatchMaker:
             print "couldn't read %s" % (contentsFilename)
             return False
 
-        self.packages = []
         xcontents = doc.FirstChildElement('contents')
         if xcontents:
             xpackage = xcontents.FirstChildElement('package')
@@ -328,10 +422,14 @@ class PatchMaker:
         """ Returns a shared PackageVersion object for the indicated
         key. """
 
-        pv = self.packageVersions.get(key, None)
+        packageName, platform, version, host, file = key
+
+        # We actually key on the hash, not the FileSpec itself.
+        k = (packageName, platform, version, host, file.hash)
+        pv = self.packageVersions.get(k, None)
         if not pv:
             pv = self.PackageVersion(*key)
-            self.packageVersions[key] = pv
+            self.packageVersions[k] = pv
         return pv
     
     def buildPatchChains(self):
@@ -341,6 +439,10 @@ class PatchMaker:
         self.patchFilenames = {}
 
         for package in self.packages:
+            if not package.baseFile:
+                # This package doesn't have any versions yet.
+                continue
+            
             currentPv = self.getPackageVersion(package.getCurrentKey())
             package.currentPv = currentPv
             currentPv.packageCurrent = package
@@ -356,15 +458,27 @@ class PatchMaker:
         """ Adds the indicated patchfile to the patch chains. """
         self.patchFilenames[patchfile.file.filename] = patchfile
 
-        fromPv = self.getPackageVersion(patchfile.getFromKey())
+        fromPv = self.getPackageVersion(patchfile.getSourceKey())
         patchfile.fromPv = fromPv
         fromPv.toPatches.append(patchfile)
 
-        toPv = self.getPackageVersion(patchfile.getToKey())
+        toPv = self.getPackageVersion(patchfile.getTargetKey())
         patchfile.toPv = toPv
         toPv.fromPatches.append(patchfile)
 
-    def processPackages(self):
+    def processSomePackages(self, packageNames):
+        """ Builds missing patches only for the named packages. """
+
+        remainingNames = packageNames[:]
+        for package in self.packages:
+            if package.packageName in remainingNames:
+                self.processPackage(package)
+                remainingNames.remove(package.packageName)
+
+        if remainingNames:
+            print "Unknown packages: %s" % (remainingNames,)
+
+    def processAllPackages(self):
         """ Walks through the list of packages, and builds missing
         patches for each one. """
 
@@ -374,6 +488,10 @@ class PatchMaker:
     def processPackage(self, package):
         """ Builds missing patches for the indicated package. """
 
+        if not package.baseFile:
+            # No versions.
+            return
+
         # Starting with the package base, how far forward can we go?
         currentPv = package.currentPv
         basePv = package.basePv
@@ -396,8 +514,8 @@ class PatchMaker:
 
     def buildPatch(self, v1, v2, package, patchFilename):
         """ Builds a patch from PackageVersion v1 to PackageVersion
-        v2, and stores it in patchFilename.  Returns true on success,
-        false on failure."""
+        v2, and stores it in patchFilename.pz.  Returns true on
+        success, false on failure."""
 
         f1 = v1.getFile()
         f2 = v2.getFile()
@@ -406,9 +524,15 @@ class PatchMaker:
         if not self.buildPatchFile(f1, f2, pathname):
             return False
 
+        compressedPathname = Filename(pathname + '.pz')
+        compressedPathname.unlink()
+        if not compressFile(pathname, compressedPathname, 9):
+            raise StandardError, "Couldn't compress patch."
+        pathname.unlink()
+
         patchfile = self.Patchfile(package)
-        patchfile.fromFile(package.packageDir, patchFilename,
-                           v1.hash, v2.hash)
+        patchfile.fromFile(package.packageDir, patchFilename + '.pz',
+                           v1.file, v2.file)
         package.patches.append(patchfile)
         package.anyChanges = True
         
@@ -425,20 +549,12 @@ class PatchMaker:
             # No original version to patch from.
             return False
 
-        print "Building patch from %s to %s" % (origFilename, newFilename)
+        print "Building patch to %s" % (newFilename)
         patchFilename.unlink()
-        p = Patchfile()
+        p = Patchfile()  # The C++ class
         if p.build(origFilename, newFilename, patchFilename):
             return True
 
         # Unable to build a patch for some reason.
         patchFilename.unlink()
         return False
-
-
-        
-if __name__ == '__main__':
-    pm = PatchMaker(Filename('install'))
-    result = pm.run()
-    print "run returned %s" % (result)
-    

direct/src/p3d/ppackage.py

@@ -28,8 +28,8 @@ Required:
 
   package.pdef
     The config file that describes the contents of the package file(s)
-    to be built, in excruciating detail.  Use "%(prog)s -H" to
-    describe the syntax of this file.
+    to be built, in excruciating detail.  See the Panda3D manual for
+    the syntax of this file.
 
 Options:
 
@@ -41,6 +41,19 @@ Options:
      copy this directory structure to a server, which will have the
      URL specified by -u, below.
 
+  -p
+     Automatically build patches against previous versions after
+     generating the results.  Patches are difference files that users
+     can download when updating a package, in lieu of redownloading
+     the whole package; this happens automatically if patches are
+     present.  You should generally build patches when you are
+     committing to a final, public-facing release.  Patches are
+     usually a good idea, but generating a patch for each internal
+     test build may needlessly generate a lot of small, inefficient
+     patch files instead of a few larger ones.  You can also generate
+     patches after the fact, by running ppatcher on the install
+     directory.
+
   -s search_dir
      Additional directories to search for previously-built packages.
      This option may be repeated as necessary.  These directories may
@@ -70,14 +83,11 @@ Options:
      appearing within the pdef file.  This information is written to
      the contents.xml file at the top of the install directory.
 
-  -p platform
+  -P platform
      Specify the platform to masquerade as.  The default is whatever
      platform Panda has been built for.  It is probably unwise to set
      this, unless you know what you are doing.
 
-  -H
-     Describe the syntax of the package.pdef input file.
-
   -h
      Display this help
 """
@@ -95,20 +105,23 @@ def usage(code, msg = ''):
     sys.exit(code)
 
 packager = Packager.Packager()
+buildPatches = False
 
 try:
-    opts, args = getopt.getopt(sys.argv[1:], 'i:s:d:p:u:n:Hh')
+    opts, args = getopt.getopt(sys.argv[1:], 'i:ps:d:P:u:n:h')
 except getopt.error, msg:
     usage(1, msg)
 
 for opt, arg in opts:
     if opt == '-i':
         packager.installDir = Filename.fromOsSpecific(arg)
+    elif opt == '-p':
+        buildPatches = True
     elif opt == '-s':
         packager.installSearch.appendDirectory(Filename.fromOsSpecific(arg))
     elif opt == '-d':
         packager.persistDir = Filename.fromOsSpecific(arg)
-    elif opt == '-p':
+    elif opt == '-P':
         packager.platform = arg
     elif opt == '-u':
         packager.host = arg
@@ -117,9 +130,6 @@ for opt, arg in opts:
         
     elif opt == '-h':
         usage(0)
-    elif opt == '-H':
-        print 'Not yet implemented.'
-        sys.exit(1)
     else:
         print 'illegal option: ' + flag
         sys.exit(1)
@@ -140,6 +150,9 @@ try:
     packager.setup()
     packages = packager.readPackageDef(packageDef)
     packager.close()
+    if buildPatches:
+        packager.buildPatches(packages)
+        
 except Packager.PackagerError:
     # Just print the error message and exit gracefully.
     inst = sys.exc_info()[1]

direct/src/p3d/ppatcher.py

@@ -0,0 +1,98 @@
+#! /usr/bin/env python
+
+"""
+
+This script generates the patches required to support incremental
+download of Panda3D packages.  It can be run as a post-process on a
+directory hierarchy created by ppackage; it will examine the directory
+hierarchy, and create any patches that appear to be missing.
+
+You may run ppackage on the same directory hierarchy as many times as
+you like, without creating patches.  You may then download and test
+the resulting files--users connecting to the tree without fresh
+patches will be forced to download the entire file, instead of making
+an incremental download, but the entire process will work otherwise.
+When you are satisfied that all of the files are ready to be released,
+you may run ppackage on the directory hierarchy to generate the
+required patches.
+
+Generating the patches just before final release is a good idea to
+limit the number of trivially small patches that are created.  Each
+time this script is run, a patch is created from the previous version,
+and these patches daisy-chain together to define a complete update
+sequence.  If you run this script on internal releases, you will
+generate a long chain of small patches that your users must download;
+this is pointless if there is no possibility of anyone having
+downloaded one of the intervening versions.
+
+You can also generate patches with the -p option to ppackage, but that
+only generates patches for the specific packages built by that
+invocation of ppackage.  If you use the ppatcher script instead, it
+will generate patches for all packages (or the set of packages that
+you name specifically).
+
+This script is actually a wrapper around Panda's PatchMaker.py.
+
+Usage:
+
+  %(prog)s [opts]
+
+Options:
+
+  -i install_dir
+     The full path to the install directory.  This should be the same
+     directory named by the -i parameter to ppackage.
+
+  -p packageName
+     Generates patches for the named package only.  This may be
+     repeated to name multiple packages.  If this is omitted, all
+     packages in the directory are scanned.
+
+  -h
+     Display this help
+"""
+
+import sys
+import getopt
+import os
+
+from direct.p3d.PatchMaker import PatchMaker
+from pandac.PandaModules import *
+
+def usage(code, msg = ''):
+    print >> sys.stderr, __doc__ % {'prog' : os.path.split(sys.argv[0])[1]}
+    print >> sys.stderr, msg
+    sys.exit(code)
+
+try:
+    opts, args = getopt.getopt(sys.argv[1:], 'i:p:h')
+except getopt.error, msg:
+    usage(1, msg)
+
+installDir = None
+packageNames = []
+
+for opt, arg in opts:
+    if opt == '-i':
+        installDir = Filename.fromOsSpecific(arg)
+    elif opt == '-p':
+        packageNames.append(arg)
+        
+    elif opt == '-h':
+        usage(0)
+    else:
+        print 'illegal option: ' + flag
+        sys.exit(1)
+
+if args:
+    usage(1)
+
+if not installDir:
+    installDir = Filename('install')
+
+if not packageNames:
+    # "None" means all packages.
+    packageNames = None
+
+pm = PatchMaker(installDir)
+pm.buildPatches(packageNames = packageNames)