integrate make_contents into Packager

direct/src/p3d/FileSpec.py

@@ -1,5 +1,5 @@
 import os
-from pandac.PandaModules import Filename, HashVal
+from pandac.PandaModules import Filename, HashVal, VirtualFileSystem
 
 class FileSpec:
     """ This class represents a disk file whose hash and size
@@ -7,7 +7,31 @@ class FileSpec:
     verify whether the file on disk matches the version demanded by
     the xml. """
 
-    def __init__(self, xelement):
+    def __init__(self):
+        pass
+
+    def fromFile(self, packageDir, filename):
+        """ Reads the file information from the indicated file. """
+        vfs = VirtualFileSystem.getGlobalPtr()
+
+        filename = Filename(filename)
+        pathname = Filename(packageDir, filename)
+        
+        self.filename = filename.cStr()
+        self.basename = filename.getBasename()
+
+        st = os.stat(pathname.toOsSpecific())
+        self.size = st.st_size
+        self.timestamp = st.st_mtime
+
+        hv = HashVal()
+        hv.hashFile(pathname)
+        self.hash = hv.asHex()
+
+    def loadXml(self, xelement):
+        """ Reads the file information from the indicated XML
+        element. """
+        
         self.filename = xelement.Attribute('filename')
         self.basename = Filename(self.filename).getBasename()
         size = xelement.Attribute('size')
@@ -23,6 +47,15 @@ class FileSpec:
             self.timestamp = 0
 
         self.hash = xelement.Attribute('hash')
+
+    def storeXml(self, xelement):
+        """ Adds the file information to the indicated XML
+        element. """
+
+        xelement.SetAttribute('filename', self.filename)
+        xelement.SetAttribute('size', str(self.size))
+        xelement.SetAttribute('timestamp', str(self.timestamp))
+        xelement.SetAttribute('hash', self.hash)
             
     def quickVerify(self, packageDir = None, pathname = None):
         """ Performs a quick test to ensure the file has not been

direct/src/p3d/HostInfo.py

@@ -58,7 +58,8 @@ class HostInfo:
             platform = xpackage.Attribute('platform')
             version = xpackage.Attribute('version')
             package = self.__makePackage(name, platform, version)
-            package.descFile = FileSpec(xpackage)
+            package.descFile = FileSpec()
+            package.descFile.loadXml(xpackage)
 
             xpackage = xpackage.NextSiblingElement('package')
 
@@ -69,7 +70,8 @@ class HostInfo:
             platform = ximport.Attribute('platform')
             version = ximport.Attribute('version')
             package = self.__makePackage(name, platform, version)
-            package.importDescFile = FileSpec(ximport)
+            package.importDescFile = FileSpec()
+            package.importDescFile.loadXml(ximport)
 
             ximport = ximport.NextSiblingElement('import')
 

direct/src/p3d/Packager.py

@@ -10,6 +10,7 @@ import marshal
 import new
 import string
 import types
+from direct.p3d.FileSpec import FileSpec
 from direct.showbase import Loader
 from direct.showbase import AppRunnerGlobal
 from direct.showutil import FreezeTool
@@ -130,6 +131,56 @@ class Packager:
             else:
                 return self.glob.matches(filename.cStr())
 
+    class PackageEntry:
+        """ This corresponds to an entry in the contents.xml file. """
+        
+        def __init__(self):
+            pass
+
+        def getKey(self):
+            """ Returns a tuple used for sorting the PackageEntry
+            objects uniquely per package. """
+            return (self.packageName, self.platform, self.version)
+
+        def fromFile(self, packageName, platform, version, solo, isImport,
+                     installDir, descFilename):
+            self.packageName = packageName
+            self.platform = platform
+            self.version = version
+            self.solo = solo
+            self.isImport = isImport
+
+            self.descFile = FileSpec()
+            self.descFile.fromFile(installDir, descFilename)
+
+        def loadXml(self, xelement):
+            self.packageName = xelement.Attribute('name')
+            self.platform = xelement.Attribute('platform')
+            self.version = xelement.Attribute('version')
+            solo = xelement.Attribute('solo')
+            self.solo = int(solo or '0')
+            self.isImport = (xelement.Value() == 'import')
+
+            self.descFile = FileSpec()
+            self.descFile.loadXml(xelement)
+
+        def makeXml(self):
+            """ Returns a new TiXmlElement. """
+            value = 'package'
+            if self.isImport:
+                value = 'import'
+            xelement = TiXmlElement(value)
+            xelement.SetAttribute('name', self.packageName)
+            if self.platform:
+                xelement.SetAttribute('platform', self.platform)
+            if self.version:
+                xelement.SetAttribute('version', self.version)
+            if self.solo:
+                xelement.SetAttribute('solo', '1')
+
+            self.descFile.storeXml(xelement)
+            return xelement
+
     class Package:
         def __init__(self, packageName, packager):
             self.packageName = packageName
@@ -408,13 +459,26 @@ class Packager:
                 self.writeDescFile()
                 self.writeImportDescFile()
 
+                # Replace or add the entry in the contents.
+                a = Packager.PackageEntry()
+                a.fromFile(self.packageName, self.platform, self.version,
+                           False, False, self.packager.installDir,
+                           self.packageDesc)
+
+                b = Packager.PackageEntry()
+                b.fromFile(self.packageName, self.platform, self.version,
+                           False, True, self.packager.installDir,
+                           self.packageImportDesc)
+                self.packager.contents[a.getKey()] = [a, b]
+                self.packager.contentsChanged = True
+
             self.cleanup()
 
         def installSolo(self):
             """ Installs the package as a "solo", which means we
-            simply copy all files into the install directory.  This is
-            primarily intended for the "coreapi" plugin, which is just
-            a single dll and a jpg file; but it can support other
+            simply copy the one file into the install directory.  This
+            is primarily intended for the "coreapi" plugin, which is
+            just a single dll and a jpg file; but it can support other
             kinds of similar "solo" packages as well. """
 
             packageDir = self.packageName
@@ -430,22 +494,38 @@ class Packager:
                 for origFile in origFiles:
                     origFile.getFilename().unlink()
 
-            if not self.files:
-                # No files, never mind.
-                return
-            
-            Filename(installPath, '').makeDir()
-            
+            files = []
             for file in self.files:
                 if file.isExcluded(self):
                     # Skip this file.
                     continue
-                targetPath = Filename(installPath, file.newName)
-                targetPath.setBinary()
-                file.filename.setBinary()
-                if not file.filename.copyTo(targetPath):
-                    print "Could not copy %s to %s" % (
-                        file.filename, targetPath)
+                files.append(file)
+
+            if not files:
+                # No files, never mind.
+                return
+
+            if len(files) != 1:
+                raise PackagerError, 'Multiple files in "solo" package %s' % (self.packageName)
+            
+            Filename(installPath, '').makeDir()
+
+            file = files[0]
+            targetPath = Filename(installPath, file.newName)
+            targetPath.setBinary()
+            file.filename.setBinary()
+            if not file.filename.copyTo(targetPath):
+                print "Could not copy %s to %s" % (
+                    file.filename, targetPath)
+
+
+            # Replace or add the entry in the contents.
+            a = Packager.PackageEntry()
+            a.fromFile(self.packageName, self.platform, self.version,
+                       True, False, self.packager.installDir,
+                       Filename(packageDir, file.newName))
+            self.packager.contents[a.getKey()] = [a]
+            self.packager.contentsChanged = True
 
             self.cleanup()
                
@@ -1106,6 +1186,7 @@ class Packager:
                     for moduleName, mdef in p2.moduleNames.items():
                         self.skipModules[moduleName] = mdef
 
+    # Packager constructor
     def __init__(self):
 
         # The following are config settings that the caller may adjust
@@ -1268,6 +1349,10 @@ class Packager:
         # A table of all known packages by name.
         self.packages = {}
 
+        # A list of PackageEntry objects read from the contents.xml
+        # file.
+        self.contents = {}
+
     def addWindowsSearchPath(self, searchPath, varname):
         """ Expands $varname, interpreting as a Windows-style search
         path, and adds its contents to the indicated DSearchPath. """
@@ -1303,133 +1388,13 @@ class Packager:
         if not PandaSystem.getPackageVersionString() or not PandaSystem.getPackageHostUrl():
             raise PackagerError, 'This script must be run using a version of Panda3D that has been built\nfor distribution.  Try using ppackage.p3d or packp3d.p3d instead.'
 
-    def __expandVariable(self, line, p):
-        """ Given that line[p] is a dollar sign beginning a variable
-        reference, advances p to the first dollar sign following the
-        reference, and looks up the variable referenced.
-
-        Returns (value, p) where value is the value of the named
-        variable, and p is the first character following the variable
-        reference. """
-
-        p += 1
-        if p >= len(line):
-            return '', p
-        
-        var = ''
-        if line[p] == '{':
-            # Curly braces exactly delimit the variable name.
-            p += 1
-            while p < len(line) and line[p] != '}':
-                var += line[p]
-                p += 1
-        else:
-            # Otherwise, a string of alphanumeric characters,
-            # including underscore, delimits the variable name.
-            var += line[p]
-            p += 1
-            while p < len(line) and (line[p] in string.letters or line[p] in string.digits or line[p] == '_'):
-                var += line[p]
-                p += 1
+        self.readContentsFile()
 
-        return ExecutionEnvironment.getEnvironmentVariable(var), p
-        
+    def close(self):
+        """ Called after reading all of the package def files, this
+        performs any final cleanup appropriate. """
 
-    def __splitLine(self, line):
-        """ Separates the indicated line into words at whitespace.
-        Quotation marks and escape characters protect spaces.  This is
-        designed to be similar to the algorithm employed by the Unix
-        shell. """
-
-        words = []
-
-        p = 0
-        while p < len(line):
-            if line[p] == '#':
-                # A word that begins with a hash mark indicates an
-                # inline comment, and the end of the parsing.
-                break
-                
-            # Scan to the end of the word.
-            word = ''
-            while p < len(line) and line[p] not in string.whitespace:
-                if line[p] == '\\':
-                    # Output an escaped character.
-                    p += 1
-                    if p < len(line):
-                        word += line[p]
-                        p += 1
-
-                elif line[p] == '$':
-                    # Output a variable reference.
-                    expand, p = self.__expandVariable(line, p)
-                    word += expand
-
-                elif line[p] == '"':
-                    # Output a double-quoted string.
-                    p += 1
-                    while p < len(line) and line[p] != '"':
-                        if line[p] == '\\':
-                            # Output an escaped character.
-                            p += 1
-                            if p < len(line):
-                                word += line[p]
-                                p += 1
-                        elif line[p] == '$':
-                            # Output a variable reference.
-                            expand, p = self.__expandVariable(line, p)
-                            word += expand
-                        else:
-                            word += line[p]
-                            p += 1
-
-                elif line[p] == "'":
-                    # Output a single-quoted string.  Escape
-                    # characters and dollar signs within single quotes
-                    # are not special.
-                    p += 1
-                    while p < len(line) and line[p] != "'":
-                        word += line[p]
-                        p += 1
-
-                else:
-                    # Output a single character.
-                    word += line[p]
-                    p += 1
-
-            words.append(word)
-
-            # Scan to the beginning of the next word.
-            while p < len(line) and line[p] in string.whitespace:
-                p += 1
-
-        return words
-
-    def __getNextLine(self):
-        """ Extracts the next line from self.inFile, and splits it
-        into words.  Returns the list of words, or None at end of
-        file. """
-
-        line = self.inFile.readline()
-        self.lineNum += 1
-        while line:
-            line = line.strip()
-            if not line:
-                # Skip the line, it was just a blank line
-                pass
-            
-            elif line[0] == '#':
-                # Eat python-style comments.
-                pass
-
-            else:
-                return self.__splitLine(line)
-
-            line = self.inFile.readline()
-            self.lineNum += 1
-
-        # End of file.
-        return None
+        self.writeContentsFile()
 
     def readPackageDef(self, packageDef):
         """ Reads the named .pdef file and constructs the packages
@@ -1568,7 +1533,7 @@ class Packager:
             self.host = hostUrl
 
         # The descriptive name, if specified, is kept until the end,
-        # where it may be passed to make_contents by ppackage.py.
+        # where it may be written into the contents file.
         if descriptiveName:
             self.hostDescriptiveName = descriptiveName
 
@@ -2188,6 +2153,59 @@ class Packager:
                                             explicit = False)
 
 
+    def readContentsFile(self):
+        """ Reads the contents.xml file at the beginning of
+        processing. """
+
+        self.contents = {}
+        self.contentsChanged = False
+
+        contentsFilename = Filename(self.installDir, 'contents.xml')
+        doc = TiXmlDocument(contentsFilename.toOsSpecific())
+        if not doc.LoadFile():
+            # Couldn't read file.
+            return
+
+        xcontents = doc.FirstChildElement('contents')
+        if xcontents:
+            if self.hostDescriptiveName is None:
+                self.hostDescriptiveName = xcontents.Attribute('descriptive_name')
+            
+            xelement = xcontents.FirstChildElement()
+            while xelement:
+                package = self.PackageEntry()
+                package.loadXml(xelement)
+                self.contents.setdefault(package.getKey(), []).append(package)
+                xelement = xelement.NextSiblingElement()
+
+    def writeContentsFile(self):
+        """ Rewrites the contents.xml file at the end of
+        processing. """
+
+        if not self.contentsChanged:
+            # No need to rewrite.
+            return
+
+        contentsFilename = Filename(self.installDir, 'contents.xml')
+        doc = TiXmlDocument(contentsFilename.toOsSpecific())
+        decl = TiXmlDeclaration("1.0", "utf-8", "")
+        doc.InsertEndChild(decl)
+
+        xcontents = TiXmlElement('contents')
+        if self.hostDescriptiveName:
+            xcontents.SetAttribute('descriptive_name', self.hostDescriptiveName)
+
+        contents = self.contents.items()
+        contents.sort()
+        for key, entryList in contents:
+            for entry in entryList:
+                xelement = entry.makeXml()
+                xcontents.InsertEndChild(xelement)
+
+        doc.InsertEndChild(xcontents)
+        doc.SaveFile()
+        
+
 # The following class and function definitions represent a few sneaky
 # Python tricks to allow the pdef syntax to contain the pseudo-Python
 # code they do.  These tricks bind the function and class definitions

direct/src/p3d/make_contents.py

@@ -1,215 +0,0 @@
-#! /usr/bin/env python
-
-"""
-This command will build the contents.xml file at the top of a Panda3D
-download hierarchy.  This file lists all of the packages hosted here,
-along with their current versions.
-
-This program runs on a local copy of the hosting directory hierarchy;
-it must be a complete copy to generate a complete contents.xml file.
-
-make_contents.py [opts]
-
-Options:
-
-  -i install_dir
-     The full path to a local directory that contains the
-     ready-to-be-published files, as populated by one or more
-     iterations of the ppackage script.  It is the user's
-     responsibility to copy this directory structure to a server.
-
-  -n "host descriptive name"
-     Specifies a descriptive name of the download server that will
-     host these contents.  This name may be presented to the user when
-     managing installed packages.  If this option is omitted, the name
-     is unchanged from the previous pass.
-
-"""
-
-import sys
-import getopt
-import os
-import types
-
-try:
-    import hashlib
-except ImportError:
-    # Legacy Python support
-    import md5 as hashlib
-
-class ArgumentError(AttributeError):
-    pass
-
-class FileSpec:
-    """ Represents a single file in the directory, and its associated
-    timestamp, size, and md5 hash. """
-    
-    def __init__(self, filename, pathname):
-        self.filename = filename
-        self.pathname = pathname
-
-        s = os.stat(pathname)
-        self.size = s.st_size
-        self.timestamp = int(s.st_mtime)
-
-        m = hashlib.md5()
-        m.update(open(pathname, 'rb').read())
-        self.hash = m.hexdigest()
-
-    def getParams(self):
-        return 'filename="%s" size="%s" timestamp="%s" hash="%s"' % (
-            self.filename, self.size, self.timestamp, self.hash)
-
-class ContentsMaker:
-    def __init__(self):
-        self.installDir = None
-        self.hostDescriptiveName = None
-
-    def build(self):
-        if not self.installDir:
-            raise ArgumentError, "Stage directory not specified."
-
-        self.packages = []
-        self.scanDirectory()
-
-        if not self.packages:
-            raise ArgumentError, "No packages found."
-
-        contentsFileBasename = 'contents.xml'
-        contentsFilePathname = os.path.join(self.installDir, contentsFileBasename)
-        contentsLine = None
-        if self.hostDescriptiveName is not None:
-            if self.hostDescriptiveName:
-                contentsLine = '<contents descriptive_name="%s">' % (
-                    self.quoteString(self.hostDescriptiveName))
-        else:
-            contentsLine = self.readContentsLine(contentsFilePathname)
-        if not contentsLine:
-            contentsLine = '<contents>'
-
-        # Now write the contents.xml file.
-        f = open(contentsFilePathname, 'w')
-        print >> f, '<?xml version="1.0" encoding="utf-8" ?>'
-        print >> f, ''
-        print >> f, contentsLine
-        for type, packageName, packagePlatform, packageVersion, file, solo in self.packages:
-            extra = ''
-            if solo:
-                extra += 'solo="1" '
-            print >> f, '  <%s name="%s" platform="%s" version="%s" %s%s />' % (
-                type, packageName, packagePlatform or '', packageVersion or '', extra, file.getParams())
-        print >> f, '</contents>'
-        f.close()
-
-    def readContentsLine(self, contentsFilePathname):
-        """ Reads the previous iteration of contents.xml to get the
-        previous top-level contents line, which contains the
-        hostDescriptiveName. """
-
-        try:
-            f = open(contentsFilePathname, 'r')
-        except IOError:
-            return None
-
-        for line in f.readlines():
-            if line.startswith('<contents'):
-                return line.rstrip()
-
-        return None
-
-    def quoteString(self, str):
-        """ Correctly quotes a string for embedding in the xml file. """
-        if isinstance(str, types.UnicodeType):
-            str = str.encode('utf-8')
-        str = str.replace('&', '&amp;')
-        str = str.replace('"', '&quot;')
-        str = str.replace('\'', '&apos;')
-        str = str.replace('<', '&lt;')
-        str = str.replace('>', '&gt;')
-        return str
-    
-    def scanDirectory(self):
-        """ Walks through all the files in the stage directory and
-        looks for the package directory xml files. """
-
-        startDir = self.installDir
-        if startDir.endswith(os.sep):
-            startDir = startDir[:-1]
-        prefix = startDir + os.sep
-        for dirpath, dirnames, filenames in os.walk(startDir):
-            if dirpath == startDir:
-                localpath = ''
-                xml = ''
-            else:
-                assert dirpath.startswith(prefix)
-                localpath = dirpath[len(prefix):].replace(os.sep, '/') + '/'
-                xml = dirpath[len(prefix):].replace(os.sep, '_') + '.xml'
-
-            solo = False
-
-            # A special case: if a directory contains just one file,
-            # it's a "solo", not an xml package.
-            if len(filenames) == 1 and not filenames[0].endswith('.xml'):
-                xml = filenames[0]
-                solo = True
-
-            if xml not in filenames:
-                continue
-
-            if localpath.count('/') == 1:
-                packageName, junk = localpath.split('/')
-                packageVersion = None
-                packagePlatform = None
-            
-            elif localpath.count('/') == 2:
-                packageName, packageVersion, junk = localpath.split('/')
-                packagePlatform = None
-
-            elif localpath.count('/') == 3:
-                packageName, packagePlatform, packageVersion, junk = localpath.split('/')
-            else:
-                continue
-
-            file = FileSpec(localpath + xml,
-                            os.path.join(self.installDir, localpath + xml))
-            print file.filename
-            self.packages.append(('package', packageName, packagePlatform, packageVersion, file, solo))
-
-            if not solo:
-                # Look for an _import.xml file, too.
-                xml = xml[:-4] + '_import.xml'
-                try:
-                    file = FileSpec(localpath + xml,
-                                    os.path.join(self.installDir, localpath + xml))
-                except OSError:
-                    file = None
-                if file:
-                    print file.filename
-                    self.packages.append(('import', packageName, packagePlatform, packageVersion, file, False))
-        
-                
-def makeContents(args):
-    opts, args = getopt.getopt(args, 'i:n:h')
-
-    cm = ContentsMaker()
-    cm.installDir = '.'
-    for option, value in opts:
-        if option == '-i':
-            cm.installDir = value
-
-        elif option == '-n':
-            cm.hostDescriptiveName = value
-            
-        elif option == '-h':
-            print __doc__
-            sys.exit(1)
-
-    cm.build()
-        
-
-if __name__ == '__main__':
-    try:
-        makeContents(sys.argv[1:])
-    except ArgumentError, e:
-        print e.args[0]
-        sys.exit(1)

direct/src/p3d/ppackage.py

@@ -87,7 +87,6 @@ import getopt
 import os
 
 from direct.p3d import Packager
-from direct.p3d import make_contents
 from pandac.PandaModules import *
 
 def usage(code, msg = ''):
@@ -112,9 +111,9 @@ for opt, arg in opts:
     elif opt == '-p':
         packager.platform = arg
     elif opt == '-u':
-        package.host = arg
+        packager.host = arg
     elif opt == '-n':
-        package.hostDescriptiveName = arg
+        packager.hostDescriptiveName = arg
         
     elif opt == '-h':
         usage(0)
@@ -140,26 +139,10 @@ packager.installSearch.prependDirectory(packager.installDir)
 try:
     packager.setup()
     packages = packager.readPackageDef(packageDef)
+    packager.close()
 except Packager.PackagerError:
     # Just print the error message and exit gracefully.
     inst = sys.exc_info()[1]
     print inst.args[0]
     #raise
     sys.exit(1)
-
-# Look to see if we built any true packages, or if all of them were
-# p3d files.
-anyPackages = False
-for package in packages:
-    if not package.p3dApplication:
-        anyPackages = True
-        break
-
-if anyPackages:
-    # If we built any true packages, then update the contents.xml at
-    # the root of the install directory.
-    cm = make_contents.ContentsMaker()
-    cm.installDir = packager.installDir.toOsSpecific()
-    cm.hostDescriptiveName = packager.hostDescriptiveName
-    cm.build()
-

direct/src/plugin/p3dHost.cxx

@@ -163,7 +163,10 @@ get_package_desc_file(FileSpec &desc_file,              // out
     const char *platform = xpackage->Attribute("platform");
     const char *version = xpackage->Attribute("version");
     const char *solo = xpackage->Attribute("solo");
-    if (name != NULL && platform != NULL && version != NULL &&
+    if (version == NULL) {
+      version = "";
+    }
+    if (name != NULL && platform != NULL &&
         package_name == name && 
         inst_mgr->get_platform() == platform &&
         package_version == version) {
@@ -190,7 +193,10 @@ get_package_desc_file(FileSpec &desc_file,              // out
     if (platform == NULL) {
       platform = "";
     }
-    if (name != NULL && version != NULL &&
+    if (version == NULL) {
+      version = "";
+    }
+    if (name != NULL &&
         package_name == name && 
         *platform == '\0' &&
         package_version == version) {