HostInfo.asMirror

direct/src/p3d/AppRunner.py

@@ -486,9 +486,7 @@ class AppRunner(DirectObject):
 
         host = self.getHost(hostUrl)
 
-        try:
-            host.readContentsFile()
-        except ValueError:
+        if not host.readContentsFile():
             if not host.downloadContentsFile(self.http):
                 print "Host %s cannot be downloaded, cannot preload %s." % (hostUrl, name)
                 return

direct/src/p3d/HostInfo.py

@@ -8,8 +8,25 @@ class HostInfo:
     Panda3D packages.  It is the Python equivalent of the P3DHost
     class in the core API. """
 
-    def __init__(self, hostUrl, appRunner):
+    def __init__(self, hostUrl, appRunner = None, hostDir = None,
+                 asMirror = False):
+
+        """ You must specify either an appRunner or a hostDir to the
+        HostInfo constructor.
+
+        If you pass asMirror = True, it means that this HostInfo
+        object is to be used to populate a "mirror" folder, a
+        duplicate (or subset) of the contents hosted by a server.
+        This means when you use this HostInfo to download packages, it
+        will only download the compressed archive file and leave it
+        there.  At the moment, mirror folders do not download old
+        patch files from the server. """
+        
+        assert appRunner or hostDir
+        
         self.hostUrl = hostUrl
+        self.hostDir = hostDir
+        self.asMirror = asMirror
 
         # hostUrlPrefix is the host URL, but it is guaranteed to end
         # with a slash.
@@ -39,7 +56,10 @@ class HostInfo:
         # will be filled in when the contents file is read.
         self.packages = {}
 
-        self.__determineHostDir(appRunner)
+        if appRunner:
+            self.__determineHostDir(appRunner)
+
+        assert self.hostDir
         self.importsDir = Filename(self.hostDir, 'imports')
 
     def downloadContentsFile(self, http):
@@ -76,32 +96,60 @@ class HostInfo:
         f.write(rf.getData())
         f.close()
 
-        try:
-            self.readContentsFile()
-        except ValueError:
+        if not self.readContentsFile():
             print "Failure reading %s" % (filename)
             return False
 
         return True
 
+    def redownloadContentsFile(self, http):
+        """ Downloads a new contents.xml file in case it has changed.
+        Returns true if the file has indeed changed, false if it has
+        not. """
+        assert self.hasContentsFile
+
+        url = self.hostUrlPrefix + 'contents.xml'
+        print "Redownloading %s" % (url)
+
+        # Get the hash of the original file.
+        filename = Filename(self.hostDir, 'contents.xml')
+        hv1 = HashVal()
+        hv1.hashFile(filename)
+
+        # Now download it again.
+        self.hasContentsFile = False
+        if not self.downloadContentsFile(http):
+            return False
+
+        hv2 = HashVal()
+        hv2.hashFile(filename)
+
+        if hv1 != hv2:
+            print "%s has changed." % (url)
+            return True
+        else:
+            print "%s has not changed." % (url)
+            return False
+
+
     def readContentsFile(self):
-        """ Reads the contents.xml file for this particular host.
-        Raises ValueError if the contents file is not already on disk
-        or is unreadable. """
+        """ Reads the contents.xml file for this particular host, once
+        it has been downloaded.  Returns true on success, false if the
+        contents file is not already on disk or is unreadable. """
 
         if self.hasContentsFile:
             # No need to read it again.
-            return
+            return True
 
         filename = Filename(self.hostDir, 'contents.xml')
 
         doc = TiXmlDocument(filename.toOsSpecific())
         if not doc.LoadFile():
-            raise ValueError
-
+            return False
+        
         xcontents = doc.FirstChildElement('contents')
         if not xcontents:
-            raise ValueError
+            return False
 
         # Look for our own entry in the hosts table.
         self.__findHostXml(xcontents)
@@ -112,7 +160,12 @@ class HostInfo:
             name = xpackage.Attribute('name')
             platform = xpackage.Attribute('platform')
             version = xpackage.Attribute('version')
-            package = self.__makePackage(name, platform, version)
+            try:
+                solo = int(xpackage.Attribute('solo') or '')
+            except ValueError:
+                solo = False
+                
+            package = self.__makePackage(name, platform, version, solo)
             package.descFile = FileSpec()
             package.descFile.loadXml(xpackage)
             package.setupFilenames()
@@ -127,6 +180,8 @@ class HostInfo:
 
         self.hasContentsFile = True
 
+        return True
+
     def __findHostXml(self, xcontents):
         """ Looks for the <host> or <alt_host> entry in the
         contents.xml that corresponds to the URL that we actually
@@ -175,7 +230,7 @@ class HostInfo:
                 self.altHosts[keyword] = url
             xalthost = xalthost.NextSiblingElement('alt_host')
 
-    def __makePackage(self, name, platform, version):
+    def __makePackage(self, name, platform, version, solo):
         """ Creates a new PackageInfo entry for the given name,
         version, and platform.  If there is already a matching
         PackageInfo, returns it. """
@@ -188,7 +243,8 @@ class HostInfo:
         platforms = self.packages.setdefault((name, version), {})
         package = platforms.get(platform, None)
         if not package:
-            package = PackageInfo(self, name, version, platform = platform)
+            package = PackageInfo(self, name, version, platform = platform,
+                                  solo = solo, asMirror = self.asMirror)
             platforms[platform] = package
 
         return package
@@ -217,19 +273,19 @@ class HostInfo:
 
         return package
 
-    def getPackages(self, name, platform = None):
+    def getPackages(self, name = None, platform = None):
         """ Returns a list of PackageInfo objects that match the
         indicated name and/or platform, with no particular regards to
-        version. """
+        version.  If name is None, all packages are returned. """
 
         assert self.hasContentsFile
 
         packages = []
         for (pn, version), platforms in self.packages.items():
-            if pn != name:
+            if name and pn != name:
                 continue
 
-            package = self.getPackage(name, version, platform = platform)
+            package = self.getPackage(pn, version, platform = platform)
             if package:
                 packages.append(package)
 

direct/src/p3d/PackageInfo.py

@@ -19,6 +19,13 @@ class PackageInfo:
     unpackFactor = 0.01
     patchFactor = 0.01
 
+    class RestartDownload(Exception):
+        """ This exception is raised by __downloadFile() when the
+        toplevel contents.xml file has changed during the download,
+        and we have to restart from the beginning. """
+        pass
+        
+
     class InstallStep:
         """ This class is one step of the installPlan list; it
         represents a single atomic piece of the installation step, and
@@ -41,16 +48,25 @@ class PackageInfo:
                 return 1
             return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
     
-    def __init__(self, host, packageName, packageVersion, platform = None):
+    def __init__(self, host, packageName, packageVersion, platform = None,
+                 solo = False, asMirror = False):
         self.host = host
         self.packageName = packageName
         self.packageVersion = packageVersion
         self.platform = platform
+        self.solo = solo
+        self.asMirror = asMirror
 
         self.packageDir = Filename(host.hostDir, self.packageName)
         if self.packageVersion:
             self.packageDir = Filename(self.packageDir, self.packageVersion)
 
+        if self.asMirror:
+            # The server directory contains the platform name, though
+            # the client directory doesn't.
+            if self.platform:
+                self.packageDir = Filename(self.packageDir, self.platform)
+            
         # These will be filled in by HostInfo when the package is read
         # from contents.xml.
         self.descFile = None
@@ -126,16 +142,17 @@ class PackageInfo:
         if self.hasPackage:
             return True
 
-        if not self.hasDescFile:
-            filename = Filename(self.packageDir, self.descFileBasename)
-            if self.descFile.quickVerify(self.packageDir, pathname = filename):
-                self.readDescFile()
-                if self.hasDescFile:
-                    # Successfully read.  We don't need to call
-                    # checkArchiveStatus again, since readDescFile()
-                    # has just done it.
-                    self.hasPackage = True
-                    return True
+        try:
+            if not self.hasDescFile:
+                filename = Filename(self.packageDir, self.descFileBasename)
+                if self.descFile.quickVerify(self.packageDir, pathname = filename):
+                    if self.__readDescFile():
+                        # Successfully read.  We don't need to call
+                        # checkArchiveStatus again, since readDescFile()
+                        # has just done it.
+                        return self.hasPackage
+        except self.RestartDownload:
+            return self.checkStatus()
 
         if self.hasDescFile:
             if self.__checkArchiveStatus():
@@ -156,33 +173,47 @@ class PackageInfo:
             return True
 
         self.http = http
-        if not self.__downloadFile(
-            None, self.descFile,
-            urlbase = self.descFile.filename,
-            filename = self.descFileBasename):
-            # Couldn't download the desc file.
-            return False
 
-        filename = Filename(self.packageDir, self.descFileBasename)
-        # Now that we've written the desc file, make it read-only.
-        os.chmod(filename.toOsSpecific(), 0444)
+        try:
+            if not self.__downloadFile(
+                None, self.descFile,
+                urlbase = self.descFile.filename,
+                filename = self.descFileBasename):
+                # Couldn't download the desc file.
+                return False
 
-        if not self.readDescFile():
-            # Weird, it passed the hash check, but we still can't read
-            # it.
-            print "Failure reading %s" % (filename)
-            return False
+            filename = Filename(self.packageDir, self.descFileBasename)
+            # Now that we've written the desc file, make it read-only.
+            os.chmod(filename.toOsSpecific(), 0444)
+
+            if not self.__readDescFile():
+                # Weird, it passed the hash check, but we still can't read
+                # it.
+                print "Failure reading %s" % (filename)
+                return False
+
+        except self.RestartDownload:
+            return self.downloadDescFile(http)
 
         return True
 
-    def readDescFile(self):
-        """ Reads the desc xml file for this particular package.
-        Returns true on success, false on failure. """
+    def __readDescFile(self):
+        """ Reads the desc xml file for this particular package,
+        assuming it's been already downloaded and verified.  Returns
+        true on success, false on failure. """
 
         if self.hasDescFile:
             # No need to read it again.
             return True
 
+        if self.solo:
+            # If this is a "solo" package, we don't actually "read"
+            # the desc file; that's the entire contents of the
+            # package.
+            self.hasDescFile = True
+            self.hasPackage = True
+            return True
+
         filename = Filename(self.packageDir, self.descFileBasename)
 
         doc = TiXmlDocument(filename.toOsSpecific())
@@ -230,15 +261,36 @@ class PackageInfo:
         # Now that we've read the desc file, go ahead and use it to
         # verify the download status.
         if self.__checkArchiveStatus():
-            # It's all good.
+            # It's all fully downloaded, unpacked, and ready.
             self.hasPackage = True
             return True
 
-        # Now set up to download the update.
+        # Still have to download it.
+        self.__buildInstallPlan()
+        return True
+
+    def __buildInstallPlan(self):
+        """ Sets up self.installPlans, a list of one or more "plans"
+        to download and install the package. """
+
         self.hasPackage = False
 
-        # Now determine what we will need to download, and build a
-        # plan (or two) to download it all.
+        if self.asMirror:
+            # If we're just downloading a mirror archive, we only need
+            # to get the compressed archive file.
+
+            # Build a one-item install plan to download the compressed
+            # archive.
+            downloadSize = self.compressedArchive.size
+            func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec)
+            
+            step = self.InstallStep(func, downloadSize, self.downloadFactor)
+            installPlan = [step]
+            self.installPlans = [installPlan]
+            return 
+
+        # The normal download process.  Determine what we will need to
+        # download, and build a plan (or two) to download it all.
         self.installPlans = None
 
         # We know we will at least need to unpackage the archive at
@@ -254,7 +306,7 @@ class PackageInfo:
         self.uncompressedArchive.actualFile = None
         if self.uncompressedArchive.quickVerify(self.packageDir):
             self.installPlans = [planA]
-            return True
+            return
 
         # Maybe the compressed archive file is good.
         if self.compressedArchive.quickVerify(self.packageDir):
@@ -262,7 +314,7 @@ class PackageInfo:
             step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
             planA = [step] + planA
             self.installPlans = [planA]
-            return True
+            return
 
         # Maybe we can download one or more patches.  We'll come back
         # to that in a minute as plan A.  For now, construct plan B,
@@ -301,8 +353,6 @@ class PackageInfo:
             # plan B as the only plan.
             self.installPlans = [planB]
 
-        return True
-
     def __scanDirectoryRecursively(self, dirname):
         """ Generates a list of Filename objects: all of the files
         (not directories) within and below the indicated dirname. """
@@ -332,11 +382,13 @@ class PackageInfo:
 
         # Get a list of all of the files in the directory, so we can
         # remove files that don't belong.
-        contents = self.__scanDirectoryRecursively(self.packageDir)
-        self.__removeFileFromList(contents, self.uncompressedArchive.filename)
+        contents = self.__scanDirectoryRecursively(self.packageDir) 
         self.__removeFileFromList(contents, self.descFileBasename)
-        for file in self.extracts:
-            self.__removeFileFromList(contents, file.filename)
+        self.__removeFileFromList(contents, self.compressedArchive.filename)
+        if not self.asMirror:
+            self.__removeFileFromList(contents, self.uncompressedArchive.filename)
+            for file in self.extracts:
+                self.__removeFileFromList(contents, file.filename)
 
         # Now, any files that are still in the contents list don't
         # belong.  It's important to remove these files before we
@@ -347,6 +399,9 @@ class PackageInfo:
             print "Removing %s" % (filename)
             pathname = Filename(self.packageDir, filename)
             pathname.unlink()
+
+        if self.asMirror:
+            return self.compressedArchive.quickVerify(self.packageDir)
             
         allExtractsOk = True
         if not self.uncompressedArchive.quickVerify(self.packageDir):
@@ -354,6 +409,11 @@ class PackageInfo:
             allExtractsOk = False
 
         if allExtractsOk:
+            # OK, the uncompressed archive is good; that means there
+            # shouldn't be a compressed archive file here.
+            pathname = Filename(self.packageDir, self.compressedArchive.filename)
+            pathname.unlink()
+            
             for file in self.extracts:
                 if not file.quickVerify(self.packageDir):
                     #print "File is incorrect: %s" % (file.filename)
@@ -378,7 +438,11 @@ class PackageInfo:
     def downloadPackage(self, http):
         """ Downloads the package file, synchronously, then
         uncompresses and unpacks it.  Returns true on success, false
-        on failure. """
+        on failure.
+
+        This assumes that self.installPlans has already been filled
+        in, which will have been done by self.__readDescFile().
+        """
 
         assert self.hasDescFile
 
@@ -388,9 +452,21 @@ class PackageInfo:
 
         # We should have an install plan by the time we get here.
         assert self.installPlans
+        installPlans = self.installPlans
+        self.installPlans = None
 
+        try:
+            return self.__followInstallPlans(installPlans, http)
+
+        except self.RestartDownload:
+            if not self.downloadDescFile(http):
+                return False
+            return self.downloadPackage(http)
+            
+
+    def __followInstallPlans(self, installPlans, http):
         self.http = http
-        for plan in self.installPlans:
+        for plan in installPlans:
             self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan))
             self.totalPlanCompleted = 0
             self.downloadProgress = 0
@@ -502,12 +578,18 @@ class PackageInfo:
                 continue
 
             if not fileSpec.fullVerify(self.packageDir, pathname = targetPathname):
-                print "After downloading, %s incorrect" % (url)
+                print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())
                 continue
 
             return True
 
-        # All mirrors failed.
+        # All mirrors failed.  Maybe the original contents.xml file is
+        # stale.  Try re-downloading it, in desperation.
+        if self.host.redownloadContentsFile(self.http):
+            raise self.RestartDownload
+
+        # Nope, nothing's changed; the server (or the internet
+        # connection) must be just fubar.
         return False
 
     def __applyPatch(self, step, patchfile):

direct/src/p3d/PatchMaker.py

@@ -247,7 +247,7 @@ class PatchMaker:
         """ This is a particular package.  This contains all of the
         information needed to reconstruct the package's desc file. """
         
-        def __init__(self, packageDesc, patchMaker, xpackage):
+        def __init__(self, packageDesc, patchMaker, xpackage = None):
             self.packageDir = Filename(patchMaker.installDir, packageDesc.getDirname())
             self.packageDesc = packageDesc
             self.patchMaker = patchMaker
@@ -256,6 +256,13 @@ class PatchMaker:
             self.currentPv = None
             self.basePv = None
 
+            self.packageName = None
+            self.platform = None
+            self.version = None
+            self.host = None
+            self.currentFile = None
+            self.baseFile = None
+
             self.doc = None
             self.anyChanges = False
             self.patches = []
@@ -279,6 +286,7 @@ class PatchMaker:
             packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc)
             self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
             if not self.doc.LoadFile():
+                print "Couldn't read %s" % (packageDescFullpath)
                 return
             
             xpackage = self.doc.FirstChildElement('package')
@@ -420,12 +428,13 @@ class PatchMaker:
 
             self.doc.SaveFile()
 
-            # Now that we've rewritten the xml file, we have to change
-            # the contents.xml file that references it to indicate the
-            # new file hash.
-            fileSpec = FileSpec()
-            fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
-            fileSpec.storeXml(self.contentsDocPackage)
+            if self.contentsDocPackage:
+                # Now that we've rewritten the xml file, we have to
+                # change the contents.xml file that references it to
+                # indicate the new file hash.
+                fileSpec = FileSpec()
+                fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
+                fileSpec.storeXml(self.contentsDocPackage)
             
 
     def __init__(self, installDir):