extend pmerge.py to validate and/or correct hashes

direct/src/p3d/FileSpec.py

@@ -1,4 +1,5 @@
 import os
+import time
 from pandac.PandaModules import Filename, HashVal, VirtualFileSystem
 
 class FileSpec:
@@ -88,13 +89,18 @@ class FileSpec:
             xelement.SetAttribute('hash', self.hash)
             
     def quickVerify(self, packageDir = None, pathname = None,
-                    notify = None):
+                    notify = None, correctSelf = False):
         """ Performs a quick test to ensure the file has not been
         modified.  This test is vulnerable to people maliciously
         attempting to fool the program (by setting datestamps etc.).
 
-        Returns true if it is intact, false if it needs to be
-        redownloaded. """
+        if correctSelf is True, then any discrepency is corrected by
+        updating the appropriate fields internally, making the
+        assumption that the file on disk is the authoritative version.
+
+        Returns true if it is intact, false if it is incorrect.  If
+        correctSelf is true, raises OSError if the self-update is
+        impossible (for instance, because the file does not exist)."""
 
         if not pathname:
             pathname = Filename(packageDir, self.filename)
@@ -104,12 +110,16 @@ class FileSpec:
             # If the file is missing, the file fails.
             if notify:
                 notify.debug("file not found: %s" % (pathname))
+                if correctSelf:
+                    raise
             return False
 
         if st.st_size != self.size:
             # If the size is wrong, the file fails.
             if notify:
                 notify.debug("size wrong: %s" % (pathname))
+            if correctSelf:
+                self.__correctHash(packageDir, pathname, st, notify)
             return False
 
         if st.st_mtime == self.timestamp:
@@ -129,6 +139,8 @@ class FileSpec:
             if notify:
                 notify.debug("hash check wrong: %s" % (pathname))
                 notify.debug("  found %s, expected %s" % (self.actualFile.hash, self.hash))
+            if correctSelf:
+                self.__correctHash(packageDir, pathname, st, notify)
             return False
 
         if notify:
@@ -137,7 +149,12 @@ class FileSpec:
         # The hash is OK after all.  Change the file's timestamp back
         # to what we expect it to be, so we can quick-verify it
         # successfully next time.
-        self.__updateTimestamp(pathname, st)
+        if correctSelf:
+            # Or update our own timestamp.
+            self.__correctTimestamp(pathname, st, notify)
+            return False
+        else:
+            self.__updateTimestamp(pathname, st)
 
         return True
         
@@ -194,6 +211,14 @@ class FileSpec:
         except OSError:
             pass
 
+    def __correctTimestamp(self, pathname, st, notify):
+        """ Corrects the internal timestamp to match the one on
+        disk. """
+        if notify:
+            notify.info("Correcting timestamp of %s to %d (%s)" % (
+                self.filename, st.st_mtime, time.asctime(time.localtime(st.st_mtime))))
+        self.timestamp = st.st_mtime
+
     def checkHash(self, packageDir, pathname, st):
         """ Returns true if the file has the expected md5 hash, false
         otherwise.  As a side effect, stores a FileSpec corresponding
@@ -205,4 +230,15 @@ class FileSpec:
         self.actualFile = fileSpec
 
         return (fileSpec.hash == self.hash)
-    
+
+    def __correctHash(self, packageDir, pathname, st, notify):
+        """ Corrects the internal hash to match the one on disk. """
+        if not self.actualFile:
+            self.checkHash(packageDir, pathname, st)
+            
+        if notify:
+            notify.info("Correcting hash %s to %s" % (
+                self.filename, self.actualFile.hash))
+        self.hash = self.actualFile.hash
+        self.size = self.actualFile.size
+        self.timestamp = self.actualFile.timestamp

direct/src/p3d/PackageMerger.py

@@ -1,5 +1,6 @@
 from direct.p3d.FileSpec import FileSpec
 from direct.p3d.SeqValue import SeqValue
+from direct.directnotify.DirectNotifyGlobal import *
 from pandac.PandaModules import *
 import copy
 import shutil
@@ -15,6 +16,8 @@ class PackageMerger:
     hosts are in sync, so that the file across all builds with the
     most recent timestamp (indicated in the contents.xml file) is
     always the most current version of the file. """
+
+    notify = directNotify.newCategory("PackageMerger")
  
     class PackageEntry:
         """ This corresponds to a <package> entry in the contents.xml
@@ -42,6 +45,10 @@ class PackageMerger:
             self.descFile = FileSpec()
             self.descFile.loadXml(xpackage)
 
+            self.validatePackageContents()
+            
+            self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
+
             self.packageSeq = SeqValue()
             self.packageSeq.loadXml(xpackage, 'seq')
             self.packageSetVer = SeqValue()
@@ -52,6 +59,7 @@ class PackageMerger:
             if ximport:
                 self.importDescFile = FileSpec()
                 self.importDescFile.loadXml(ximport)
+                self.importDescFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
 
         def makeXml(self):
             """ Returns a new TiXmlElement. """
@@ -75,6 +83,50 @@ class PackageMerger:
             
             return xpackage
 
+        def validatePackageContents(self):
+            """ Validates the contents of the package directory itself
+            against the expected hashes and timestamps.  Updates
+            hashes and timestamps where needed. """
+
+            if self.solo:
+                return
+
+            needsChange = False
+            packageDescFullpath = Filename(self.sourceDir, self.descFile.filename)
+            packageDir = Filename(packageDescFullpath.getDirname())
+            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
+            if not doc.LoadFile():
+                message = "Could not read XML file: %s" % (self.descFile.filename)
+                raise OSError, message
+
+            xpackage = doc.FirstChildElement('package')
+            if not xpackage:
+                message = "No package definition: %s" % (self.descFile.filename)
+                raise OSError, message
+
+            xcompressed = xpackage.FirstChildElement('compressed_archive')
+            if xcompressed:
+                spec = FileSpec()
+                spec.loadXml(xcompressed)
+                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
+                    spec.storeXml(xcompressed)
+                    needsChange = True
+
+            xpatch = xpackage.FirstChildElement('patch')
+            while xpatch:
+                spec = FileSpec()
+                spec.loadXml(xpatch)
+                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
+                    spec.storeXml(xpatch)
+                    needsChange = True
+
+                xpatch = xpatch.NextSiblingElement('patch')
+
+            if needsChange:
+                PackageMerger.notify.info("Rewriting %s" % (self.descFile.filename))
+                doc.SaveFile()
+                self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
+
     # PackageMerger constructor
     def __init__(self, installDir):
         self.installDir = installDir
@@ -161,7 +213,7 @@ class PackageMerger:
         there. """
 
         dirname = Filename(pe.descFile.filename).getDirname()
-        print "copying %s" % (dirname)
+        self.notify.info("copying %s" % (dirname))
         sourceDirname = Filename(pe.sourceDir, dirname)
         targetDirname = Filename(self.installDir, dirname)
 
@@ -208,6 +260,13 @@ class PackageMerger:
             # Copying a regular file.
             sourceFilename.copyTo(targetFilename)
 
+            # Also try to copy the timestamp, but don't fuss too much
+            # if it doesn't work.
+            try:
+                st = os.stat(sourceFilename.toOsSpecific())
+                os.utime(targetFilename.toOsSpecific(), (st.st_atime, st.st_mtime))
+            except OSError:
+                pass
 
     def merge(self, sourceDir):
         """ Adds the contents of the indicated source directory into

direct/src/p3d/pmerge.py

@@ -4,7 +4,9 @@ usageText = """
 
 This script can be used to merge together the contents of two or more
 separately-built stage directories, built independently via ppackage,
-or via Packager.py.
+or via Packager.py.  This script also verifies the hash, file size,
+and timestamp values in the stage directory as it runs, so it can be
+run on a single standalone directory just to perform this validation.
 
 This script is actually a wrapper around Panda's PackageMerger.py.
 
@@ -24,7 +26,9 @@ Options:
   -i install_dir
      The full path to the final install directory.  This may also
      contain some pre-existing contents; if so, it is merged with all
-     of the input directories as well.
+     of the input directories as well.  The contents of this directory
+     are checked for self-consistency with regards to hashes and
+     timestamps.
 
   -h
      Display this help
@@ -62,9 +66,11 @@ inputDirs = []
 for arg in args:
     inputDirs.append(Filename.fromOsSpecific(arg))
 
-if not inputDirs:
-    print "no input directories specified."
-    sys.exit(1)
+# It's now legal to have no input files if you only want to verify
+# timestamps and hashes.
+## if not inputDirs:
+##     print "no input directories specified."
+##     sys.exit(1)
 
 try:
     pm = PackageMerger.PackageMerger(installDir)