reduce end-of-phase chugs

direct/src/p3d/PackageInfo.py

@@ -1,4 +1,4 @@
-from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment
+from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector
 from pandac import PandaModules
 from direct.p3d.FileSpec import FileSpec
 from direct.showbase import VFSImporter
@@ -37,11 +37,29 @@ class PackageInfo:
         represents a single atomic piece of the installation step, and
         the relative effort of that piece.  When the plan is executed,
         it will call the saved function pointer here. """
-        def __init__(self, func, bytes, factor):
-            self.func = func
+        def __init__(self, func, bytes, factor, stepType):
+            self.__funcPtr = func
             self.bytesNeeded = bytes
             self.bytesDone = 0
             self.bytesFactor = factor
+            self.stepType = stepType
+            self.pStatCol = PStatCollector(':App:PackageInstaller:%s' % (stepType))
+
+        def func(self):
+            """ self.__funcPtr(self) will return a generator of
+            tokens.  This function defines a new generator that yields
+            each of those tokens, but wraps each call into the nested
+            generator within a pair of start/stop collector calls. """
+            
+            self.pStatCol.start()
+            for token in self.__funcPtr(self):
+                self.pStatCol.stop()
+                yield token
+                self.pStatCol.start()
+
+            # Shouldn't ever get here.
+            self.pStatCol.stop()
+            raise StopIteration
 
         def getEffort(self):
             """ Returns the relative amount of effort of this step. """
@@ -215,10 +233,13 @@ class PackageInfo:
 
         self.http = http
 
-        for token in self.__downloadFile(
+        func = lambda step, self = self: self.__downloadFile(
             None, self.descFile,
             urlbase = self.descFile.filename,
-            filename = self.descFileBasename):
+            filename = self.descFileBasename)
+        step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')
+
+        for token in step.func():
             if token == self.stepContinue:
                 yield token
             else:
@@ -226,10 +247,12 @@ class PackageInfo:
 
         while token == self.restartDownload:
             # Try again.
-            for token in self.__downloadFile(
+            func = lambda step, self = self: self.__downloadFile(
                 None, self.descFile,
                 urlbase = self.descFile.filename,
-                filename = self.descFileBasename):
+                filename = self.descFileBasename)
+            step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')
+            for token in step.func():
                 if token == self.stepContinue:
                     yield token
                 else:
@@ -350,6 +373,9 @@ class PackageInfo:
         """ Sets up self.installPlans, a list of one or more "plans"
         to download and install the package. """
 
+        pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
+        pc.start()
+
         self.hasPackage = False
 
         if self.asMirror:
@@ -361,9 +387,10 @@ class PackageInfo:
             downloadSize = self.compressedArchive.size
             func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
             
-            step = self.InstallStep(func, downloadSize, self.downloadFactor)
+            step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
             installPlan = [step]
             self.installPlans = [installPlan]
+            pc.stop()
             return 
 
         # The normal download process.  Determine what we will need to
@@ -375,7 +402,7 @@ class PackageInfo:
         unpackSize = 0
         for file in self.extracts:
             unpackSize += file.size
-        step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor)
+        step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
         planA = [step]
 
         # If the uncompressed archive file is good, that's all we'll
@@ -383,14 +410,16 @@ class PackageInfo:
         self.uncompressedArchive.actualFile = None
         if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
             self.installPlans = [planA]
+            pc.stop()
             return
 
         # Maybe the compressed archive file is good.
         if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
             uncompressSize = self.uncompressedArchive.size
-            step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
+            step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
             planA = [step] + planA
             self.installPlans = [planA]
+            pc.stop()
             return
 
         # Maybe we can download one or more patches.  We'll come back
@@ -399,13 +428,13 @@ class PackageInfo:
         planB = planA[:]
 
         uncompressSize = self.uncompressedArchive.size
-        step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
+        step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
         planB = [step] + planB
 
         downloadSize = self.compressedArchive.size
         func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
 
-        step = self.InstallStep(func, downloadSize, self.downloadFactor)
+        step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
         planB = [step] + planB
 
         # Now look for patches.  Start with the md5 hash from the
@@ -430,6 +459,8 @@ class PackageInfo:
             # plan B as the only plan.
             self.installPlans = [planB]
 
+        pc.stop()
+
     def __scanDirectoryRecursively(self, dirname):
         """ Generates a list of Filename objects: all of the files
         (not directories) within and below the indicated dirname. """
@@ -589,7 +620,7 @@ class PackageInfo:
             for step in plan:
                 self.currentStepEffort = step.getEffort()
 
-                for token in step.func(step):
+                for token in step.func():
                     if token == self.stepContinue:
                         yield token
                     else:
@@ -633,12 +664,12 @@ class PackageInfo:
         for patchfile in patchChain:
             downloadSize = patchfile.file.size
             func = lambda step, fileSpec = patchfile.file: self.__downloadFile(step, fileSpec, allowPartial = True)
-            step = self.InstallStep(func, downloadSize, self.downloadFactor)
+            step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
             plan.append(step)
 
             patchSize = patchfile.targetFile.size
             func = lambda step, patchfile = patchfile: self.__applyPatch(step, patchfile)
-            step = self.InstallStep(func, patchSize, self.patchFactor)
+            step = self.InstallStep(func, patchSize, self.patchFactor, 'patch')
             plan.append(step)
 
         patchMaker.cleanup()

direct/src/p3d/PackageInstaller.py

@@ -3,7 +3,7 @@ from direct.stdpy.threading import Lock, RLock
 from direct.showbase.MessengerGlobal import messenger
 from direct.task.TaskManagerGlobal import taskMgr
 from direct.p3d.PackageInfo import PackageInfo
-from pandac.PandaModules import TPLow
+from pandac.PandaModules import TPLow, PStatCollector
 from direct.directnotify.DirectNotifyGlobal import directNotify
 
 class PackageInstaller(DirectObject):
@@ -495,8 +495,10 @@ class PackageInstaller(DirectObject):
             # files, stop the task.
             if not self.needsDescFile:
                 self.descFileTask = None
-                messenger.send('PackageInstaller-%s-allHaveDesc' % self.uniqueId,
-                               taskChain = 'default')
+
+                eventName = 'PackageInstaller-%s-allHaveDesc' % self.uniqueId
+                messenger.send(eventName, taskChain = 'default')
+
                 return task.done
             pp = self.needsDescFile[0]
             del self.needsDescFile[0]
@@ -545,11 +547,12 @@ class PackageInstaller(DirectObject):
                 del self.needsDownload[0]
             except:
                 self.packageLock.release()
+                raise
             self.packageLock.release()
 
             # Now serve this one package.
-            messenger.send('PackageInstaller-%s-packageStarted' % self.uniqueId,
-                           [pp], taskChain = 'default')
+            eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
+            messenger.send(eventName, [pp], taskChain = 'default')
 
             if not pp.package.hasPackage:
                 for token in pp.package.downloadPackageGenerator(self.appRunner.http):
@@ -559,12 +562,18 @@ class PackageInstaller(DirectObject):
                         break
 
                 if token != pp.package.stepComplete:
+                    pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
+                    pc.start()
                     self.__donePackage(pp, False)
+                    pc.stop()
                     yield task.cont
                     continue
 
             # Successfully downloaded and installed.
+            pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
+            pc.start()
             self.__donePackage(pp, True)
+            pc.stop()
 
             # Continue the loop without yielding, so we pick up the
             # next package within this same frame.
@@ -575,7 +584,10 @@ class PackageInstaller(DirectObject):
         assert not pp.done
 
         if success:
+            pc = PStatCollector(':App:PackageInstaller:install:%s' % (pp.package.packageName))
+            pc.start()
             pp.package.installPackage(self.appRunner)
+            pc.stop()
 
         self.packageLock.acquire()
         try:
@@ -588,8 +600,8 @@ class PackageInstaller(DirectObject):
         finally:
             self.packageLock.release()
 
-        messenger.send('PackageInstaller-%s-packageDone' % self.uniqueId,
-                       [pp], taskChain = 'default')
+        eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
+        messenger.send(eventName, [pp], taskChain = 'default')
 
     def __progressTask(self, task):
         self.callbackLock.acquire()

panda/src/downloader/config_downloader.cxx

@@ -49,12 +49,12 @@ ConfigVariableInt downloader_timeout_retries
 ("downloader-timeout-retries", 5);
 
 ConfigVariableDouble decompressor_step_time
-("decompressor-step-time", 0.1,
+("decompressor-step-time", 0.005,
  PRC_DESC("Specifies the maximum amount of time that should be consumed by "
           "a single call to Decompressor::run()."));
 
 ConfigVariableDouble extractor_step_time
-("extractor-step-time", 0.1,
+("extractor-step-time", 0.005,
  PRC_DESC("Specifies the maximum amount of time that should be consumed by "
           "a single call to Extractor::step()."));