Build .p3d tools as part of SDK build process, if host URL and distributor are specified

direct/src/p3d/AppRunner.py

@@ -436,47 +436,8 @@ class AppRunner(DirectObject):
         it downloads a new version on-the-spot.  Returns true on
         success, false on failure. """
 
-        if fileSpec.quickVerify(pathname = localPathname):
-            # It's good, keep it.
-            return True
-
         assert self.http
-
-        # It's stale, get a new one.
-        doc = None
-        if self.superMirrorUrl:
-            # Use the "super mirror" first.
-            url = core.URLSpec(self.superMirrorUrl + fileSpec.filename)
-            self.notify.info("Freshening %s" % (url))
-            doc = self.http.getDocument(url)
-
-        if not doc or not doc.isValid():
-            # Failing the super mirror, contact the actual host.
-            url = core.URLSpec(host.hostUrlPrefix + fileSpec.filename)
-            self.notify.info("Freshening %s" % (url))
-            doc = self.http.getDocument(url)
-            if not doc.isValid():
-                return False
-
-        file = Filename.temporary('', 'p3d_')
-        if not doc.downloadToFile(file):
-            # Failed to download.
-            file.unlink()
-            return False
-
-        # Successfully downloaded!
-        localPathname.makeDir()
-        if not file.renameTo(localPathname):
-            # Couldn't move it into place.
-            file.unlink()
-            return False
-
-        if not fileSpec.fullVerify(pathname = localPathname, notify = self.notify):
-            # No good after download.
-            self.notify.info("%s is still no good after downloading." % (url))
-            return False
-
-        return True
+        return host.freshenFile(self.http, fileSpec, localPathname)
 
     def scanInstalledPackages(self):
         """ Scans the hosts and packages already installed locally on

direct/src/p3d/HostInfo.py

@@ -38,8 +38,6 @@ class HostInfo:
         Note that perPlatform is also restricted by the individual
         package's specification.  """
 
-        assert appRunner or rootDir or hostDir
-
         self.__setHostUrl(hostUrl)
         self.appRunner = appRunner
         self.rootDir = rootDir
@@ -110,6 +108,52 @@ class HostInfo:
             # https-protected hostUrl, it will be the cleartext channel.
             self.downloadUrlPrefix = self.hostUrlPrefix
 
+    def freshenFile(self, http, fileSpec, localPathname):
+        """ Ensures that the localPathname is the most current version
+        of the file defined by fileSpec, as offered by host.  If not,
+        it downloads a new version on-the-spot.  Returns true on
+        success, false on failure. """
+
+        if fileSpec.quickVerify(pathname = localPathname):
+            # It's good, keep it.
+            return True
+
+        # It's stale, get a new one.
+        doc = None
+        if self.appRunner and self.appRunner.superMirrorUrl:
+            # Use the "super mirror" first.
+            url = core.URLSpec(self.appRunner.superMirrorUrl + fileSpec.filename)
+            self.notify.info("Freshening %s" % (url))
+            doc = http.getDocument(url)
+
+        if not doc or not doc.isValid():
+            # Failing the super mirror, contact the actual host.
+            url = core.URLSpec(self.hostUrlPrefix + fileSpec.filename)
+            self.notify.info("Freshening %s" % (url))
+            doc = http.getDocument(url)
+            if not doc.isValid():
+                return False
+
+        file = Filename.temporary('', 'p3d_')
+        if not doc.downloadToFile(file):
+            # Failed to download.
+            file.unlink()
+            return False
+
+        # Successfully downloaded!
+        localPathname.makeDir()
+        if not file.renameTo(localPathname):
+            # Couldn't move it into place.
+            file.unlink()
+            return False
+
+        if not fileSpec.fullVerify(pathname = localPathname, notify = self.notify):
+            # No good after download.
+            self.notify.info("%s is still no good after downloading." % (url))
+            return False
+
+        return True
+
     def downloadContentsFile(self, http, redownload = False,
                              hashVal = None):
         """ Downloads the contents.xml file for this particular host,
@@ -166,7 +210,7 @@ class HostInfo:
                     channel = http.makeChannel(False)
                     channel.getDocument(request)
                     if channel.downloadToRam(rf):
-                        self.notify.warning("Successfully downloaded %s" % (url,))
+                        self.notify.info("Successfully downloaded %s" % (url,))
                         break
                     else:
                         rf = None
@@ -367,7 +411,7 @@ class HostInfo:
             assert self.hostDir
             self.__findHostXmlForHostDir(xcontents)
 
-        if not self.hostDir:
+        if self.rootDir and not self.hostDir:
             self.hostDir = self.__determineHostDir(None, self.hostUrl)
 
         # Get the list of packages available for download and/or import.
@@ -401,7 +445,7 @@ class HostInfo:
         self.hasContentsFile = True
 
         # Now save the contents.xml file into the standard location.
-        if not self.appRunner or self.appRunner.verifyContents != self.appRunner.P3DVCNever:
+        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCNever:
             assert self.hostDir
             filename = Filename(self.hostDir, 'contents.xml')
             filename.makeDir()
@@ -474,7 +518,7 @@ class HostInfo:
             self.descriptiveName = descriptiveName
 
         hostDirBasename = xhost.Attribute('host_dir')
-        if not self.hostDir:
+        if self.rootDir and not self.hostDir:
             self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl)
 
         # Get the "download" URL, which is the source from which we

direct/src/p3d/Packager.py

@@ -12,13 +12,13 @@ import glob
 import marshal
 import string
 import types
-import getpass
 import platform
 import struct
 import subprocess
 import copy
 from direct.p3d.FileSpec import FileSpec
 from direct.p3d.SeqValue import SeqValue
+from direct.p3d.HostInfo import HostInfo
 from direct.showbase import Loader
 from direct.showbase import AppRunnerGlobal
 from direct.showutil import FreezeTool
@@ -379,7 +379,8 @@ class Packager:
             # This records the current list of modules we have added so
             # far.
             self.freezer = FreezeTool.Freezer(platform = self.packager.platform)
-            
+            self.freezer.storePythonSource = self.packager.storePythonSource
+
             # Map of extensions to files to number (ignored by dir)
             self.ignoredDirFiles = {}
 
@@ -1078,7 +1079,7 @@ class Packager:
             fpath.append(Filename("/Library/Frameworks"))
             fpath.append(Filename("/System/Library/Frameworks"))
             fpath.append(Filename("/Developer/Library/Frameworks"))
-            fpath.append(Filename("/Users/%s" % getpass.getuser(), "Library/Frameworks"))
+            fpath.append(Filename(os.path.expanduser("~"), "Library/Frameworks"))
             if "HOME" in os.environ:
                 fpath.append(Filename(os.environ["HOME"], "Library/Frameworks"))
             ffilename = Filename(library.split('.framework/', 1)[0].split('/')[-1] + '.framework')
@@ -2236,6 +2237,11 @@ class Packager:
         self.host = PandaSystem.getPackageHostUrl()
         self.addHost(self.host)
 
+        # This will be used when we're not compiling in the packaged
+        # environment.
+        self.__hostInfos = {}
+        self.http = HTTPClient.getGlobalPtr()
+
         # The maximum amount of time a client should cache the
         # contents.xml before re-querying the server, in seconds.
         self.maxAge = 0
@@ -2321,6 +2327,10 @@ class Packager:
         # any applications.
         self.allowPythonDev = False
 
+        # Set this flag to store the original Python source files,
+        # without compiling them to .pyc or .pyo.
+        self.storePythonSource = False
+
         # Fill this with a list of (certificate, chain, pkey,
         # password) tuples to automatically sign each p3d file
         # generated.
@@ -3027,17 +3037,10 @@ class Packager:
 
     def __findPackageOnHost(self, packageName, platform, version, hostUrl, requires = None):
         appRunner = AppRunnerGlobal.appRunner
-        if not appRunner:
-            # We don't download import files from a host unless we're
-            # running in a packaged environment ourselves.  It would
-            # be possible to do this, but a fair bit of work for not
-            # much gain--this is meant to be run in a packaged
-            # environment.
-            return None
 
         # Make sure we have a fresh version of the contents file.
-        host = appRunner.getHost(hostUrl)
-        if not host.downloadContentsFile(appRunner.http):
+        host = self.__getHostInfo(hostUrl)
+        if not host.downloadContentsFile(self.http):
             return None
 
         packageInfos = []
@@ -3062,23 +3065,54 @@ class Packager:
 
             # Now we've retrieved a PackageInfo.  Get the import desc file
             # from it.
-            filename = Filename(host.hostDir, 'imports/' + packageInfo.importDescFile.basename)
-            if not appRunner.freshenFile(host, packageInfo.importDescFile, filename):
-                self.notify.error("Couldn't download import file.")
-                continue
+            if host.hostDir:
+                filename = Filename(host.hostDir, 'imports/' + packageInfo.importDescFile.basename)
+            else:
+                # We're not running in the packaged environment, so download
+                # to a temporary file instead of the host directory.
+                filename = Filename.temporary('', 'import_' + packageInfo.importDescFile.basename, '.xml')
+
+            # Compatibility hack: 1.9.0 does not have host.freshenFile.
+            if not hasattr(host, 'freshenFile'):
+                if not appRunner.freshenFile(host, packageInfo.importDescFile, filename):
+                    self.notify.error("Couldn't download import file.")
+                    continue
+            else:
+                if not host.freshenFile(self.http, packageInfo.importDescFile, filename):
+                    self.notify.error("Couldn't download import file.")
+                    continue
 
             # Now that we have the import desc file, use it to load one of
             # our Package objects.
             package = self.Package('', self)
-            if not package.readImportDescFile(filename):
-                continue
+            success = package.readImportDescFile(filename)
 
-            if self.__packageIsValid(package, requires, platform):
+            if not host.hostDir:
+                # Don't forget to delete the temporary file we created.
+                filename.unlink()
+
+            if success and self.__packageIsValid(package, requires, platform):
                 return package
 
         # Couldn't find a suitable package.
         return None
 
+    def __getHostInfo(self, hostUrl = None):
+        """ This shadows appRunner.getHost(), for the purpose of running
+        outside the packaged environment. """
+
+        if not hostUrl:
+            hostUrl = PandaSystem.getPackageHostUrl()
+
+        if AppRunnerGlobal.appRunner:
+            return AppRunnerGlobal.appRunner.getHost(hostUrl)
+
+        host = self.__hostInfos.get(hostUrl, None)
+        if not host:
+            host = HostInfo(hostUrl)
+            self.__hostInfos[hostUrl] = host
+        return host
+
     def __sortImportPackages(self, packages):
         """ Given a list of Packages read from *.import.xml filenames,
         sorts them in reverse order by version, so that the

direct/src/p3d/ppackage.py

@@ -83,6 +83,10 @@ Options:
      initially, but should not be set on an application intended for
      deployment.
 
+  -N
+     If this option is set, Packager will not try to compile any Python
+     files to .pyc or .pyo, instead storing the original source files.
+
   -u
      On the Mac OSX platform, this means that Panda was built with
      universal binaries, and the package should be built that way as
@@ -152,6 +156,7 @@ buildPatches = False
 installSearch = []
 signParams = []
 allowPythonDev = False
+storePythonSource = False
 universalBinaries = False
 systemRoot = None
 ignoreSetHost = False
@@ -160,7 +165,7 @@ p3dSuffix = ''
 platforms = []
 
 try:
-    opts, args = getopt.getopt(sys.argv[1:], 'i:ps:S:DuP:R:Ha:hv')
+    opts, args = getopt.getopt(sys.argv[1:], 'i:ps:S:DNuP:R:Ha:hv')
 except getopt.error, msg:
     usage(1, msg)
 
@@ -182,6 +187,8 @@ for opt, arg in opts:
                            Filename.fromOsSpecific(password)))
     elif opt == '-D':
         allowPythonDev = True
+    elif opt == '-N':
+        storePythonSource = True
     elif opt == '-u':
         universalBinaries = True
     elif opt == '-P':
@@ -236,6 +243,7 @@ for platform in platforms:
         packager.installSearch = [installDir] + packager.installSearch
     packager.signParams = signParams
     packager.allowPythonDev = allowPythonDev
+    packager.storePythonSource = storePythonSource
     packager.systemRoot = systemRoot
     packager.ignoreSetHost = ignoreSetHost
     packager.verbosePrint = verbosePrint

direct/src/showutil/FreezeTool.py

@@ -1147,6 +1147,7 @@ class Freezer:
         elif getattr(module, '__file__', None):
             sourceFilename = Filename.fromOsSpecific(module.__file__)
             sourceFilename.setExtension("py")
+            sourceFilename.setText()
 
         if self.storePythonSource:
             if sourceFilename and sourceFilename.exists():

makepanda/makepanda.py

@@ -373,7 +373,7 @@ if RUNTIME or RTDIST:
 if DEBVERSION is None:
     DEBVERSION = VERSION
 
-MAJOR_VERSION = VERSION[:3]
+MAJOR_VERSION = '.'.join(VERSION.split('.')[:2])
 
 if P3DSUFFIX is None:
     P3DSUFFIX = MAJOR_VERSION
@@ -392,9 +392,10 @@ if (RUNTIME or RTDIST):
     if (RUNTIME):
         outputdir_suffix += "_rt"
 
-    RTDIST_VERSION = DISTRIBUTOR.strip() + "_" + MAJOR_VERSION
-elif (DISTRIBUTOR == ""):
+if DISTRIBUTOR == "":
     DISTRIBUTOR = "makepanda"
+else:
+    RTDIST_VERSION = DISTRIBUTOR.strip() + "_" + MAJOR_VERSION
 
 if not IsCustomOutputDir():
     if GetTarget() == "windows" and GetTargetArch() == 'x64':
@@ -1851,6 +1852,11 @@ def Package(target, inputs, opts):
 
     command += "direct/src/p3d/ppackage.py"
 
+    if not RTDIST:
+        # Don't compile Python sources, because we might not running in the same
+        # Python version as the selected host.
+        command += " -N"
+
     if GetTarget() == "darwin":
         if SDK.get("MACOSX"):
             command += " -R \"%s\"" % SDK["MACOSX"]
@@ -1863,8 +1869,32 @@ def Package(target, inputs, opts):
     command += " -i \"" + GetOutputDir() + "/stage\""
     if (P3DSUFFIX):
         command += ' -a "' + P3DSUFFIX + '"'
+
     command += " " + inputs[0]
-    oscmd(command)
+
+    if GetOrigExt(target) == '.p3d':
+        # Build a specific .p3d file.
+        basename = os.path.basename(os.path.splitext(target)[0])
+        command += " " + basename
+        oscmd(command)
+
+        if GetTarget() == 'windows':
+            # TODO: build p3dWrapper.c executable.
+            #objfile = FindLocation('p3dWrapper_' + basename + '.obj', [])
+            #CompileCxx(objfile, 'p3dWrapper.c', ['DIR:direct/src/p3d'])
+
+            #exefile = os.path.splitext(target)[0] + '.exe'
+            #CompileLink(exefile, [objfile], ['ADVAPI'])
+            pass
+
+        # Move it to the bin directory.
+        os.rename(GetOutputDir() + '/stage/' + basename + P3DSUFFIX + '.p3d', target)
+
+        if sys.platform != 'win32':
+            oscmd('chmod +x ' + BracketNameWithQuotes(target))
+    else:
+        # This is presumably a package or set of packages.
+        oscmd(command)
 
 ##########################################################################################
 #
@@ -1957,7 +1987,10 @@ def CompileAnything(target, inputs, opts, progress = None):
         ProgressOutput(progress, "Compiling MIDL file", infile)
         return CompileMIDL(target, infile, opts)
     elif (infile.endswith(".pdef")):
-        ProgressOutput(progress, "Building package from pdef file", infile)
+        if origsuffix == '.p3d':
+            ProgressOutput(progress, "Building package", target)
+        else:
+            ProgressOutput(progress, "Building package from pdef file", infile)
         return Package(target, inputs, opts)
     elif origsuffix in SUFFIX_LIB:
         ProgressOutput(progress, "Linking static library", target)
@@ -6187,26 +6220,17 @@ if (RTDIST):
   TargetAdd('_thirdparty', opts=OPTS, input='thirdparty.pdef')
 
 #
-# Distribute prebuilt .p3d files as executable.
+# If we have a host URL and distributor, we can make .p3d deployment tools.
 #
 
-if (PkgSkip("DIRECT")==0 and not RUNTIME and not RTDIST):
-  if GetTarget() == 'windows':
+if not PkgSkip("DIRECT") and not PkgSkip("DEPLOYTOOLS") and not RUNTIME and not RTDIST and HOST_URL and DISTRIBUTOR:
     OPTS=['DIR:direct/src/p3d']
-    TargetAdd('p3dWrapper.obj', opts=OPTS, input='p3dWrapper.c')
-    TargetAdd('p3dWrapper.exe', input='p3dWrapper.obj')
-    TargetAdd('p3dWrapper.exe', opts=["ADVAPI"])
-
-  for g in glob.glob("direct/src/p3d/*.p3d"):
-    base = os.path.basename(g)
-    base = base.split(".", 1)[0]
 
-    if GetTarget() == 'windows':
-      TargetAdd(base+".exe", input='p3dWrapper.exe')
-      CopyFile(GetOutputDir()+"/bin/"+base+".p3d", g)
-    else:
-      CopyFile(GetOutputDir()+"/bin/"+base, g)
-      oscmd("chmod +x "+GetOutputDir()+"/bin/"+base)
+    TargetAdd('packp3d.p3d', opts=OPTS, input='panda3d.pdef')
+    TargetAdd('pdeploy.p3d', opts=OPTS, input='panda3d.pdef')
+    TargetAdd('pmerge.p3d', opts=OPTS, input='panda3d.pdef')
+    TargetAdd('ppackage.p3d', opts=OPTS, input='panda3d.pdef')
+    TargetAdd('ppatcher.p3d', opts=OPTS, input='panda3d.pdef')
 
 ##########################################################################################
 #

makepanda/makepandacore.py

@@ -2762,6 +2762,7 @@ def CalcLocation(fn, ipath):
         if (fn.endswith(".dle")):   return OUTPUTDIR+"/plugins/"+fn[:-4]+dllext+".dle"
         if (fn.endswith(".plugin")):return OUTPUTDIR+"/plugins/"+fn[:-7]+dllext+".dll"
         if (fn.endswith(".exe")):   return OUTPUTDIR+"/bin/"+fn
+        if (fn.endswith(".p3d")):   return OUTPUTDIR+"/bin/"+fn
         if (fn.endswith(".lib")):   return OUTPUTDIR+"/lib/"+fn[:-4]+dllext+".lib"
         if (fn.endswith(".ilb")):   return OUTPUTDIR+"/tmp/"+fn[:-4]+dllext+".lib"
     elif (target == 'darwin'):
@@ -2773,6 +2774,7 @@ def CalcLocation(fn, ipath):
         if (fn.endswith(".pyd")):   return OUTPUTDIR+"/panda3d/"+fn[:-4]+".so"
         if (fn.endswith(".mll")):   return OUTPUTDIR+"/plugins/"+fn
         if (fn.endswith(".exe")):   return OUTPUTDIR+"/bin/"+fn[:-4]
+        if (fn.endswith(".p3d")):   return OUTPUTDIR+"/bin/"+fn[:-4]
         if (fn.endswith(".lib")):   return OUTPUTDIR+"/lib/"+fn[:-4]+".a"
         if (fn.endswith(".ilb")):   return OUTPUTDIR+"/tmp/"+fn[:-4]+".a"
         if (fn.endswith(".rsrc")):  return OUTPUTDIR+"/tmp/"+fn
@@ -2795,6 +2797,7 @@ def CalcLocation(fn, ipath):
         if (fn.endswith(".mll")):   return OUTPUTDIR+"/plugins/"+fn
         if (fn.endswith(".plugin")):return OUTPUTDIR+"/plugins/"+fn[:-7]+dllext+".so"
         if (fn.endswith(".exe")):   return OUTPUTDIR+"/bin/"+fn[:-4]
+        if (fn.endswith(".p3d")):   return OUTPUTDIR+"/bin/"+fn[:-4]
         if (fn.endswith(".lib")):   return OUTPUTDIR+"/lib/"+fn[:-4]+".a"
         if (fn.endswith(".ilb")):   return OUTPUTDIR+"/tmp/"+fn[:-4]+".a"
     if (fn.endswith(".dat")):   return OUTPUTDIR+"/tmp/"+fn