many bug fixes, almost completed

direct/src/showbase/ContainerLeakDetector.py

@@ -2,7 +2,15 @@ from direct.directnotify.DirectNotifyGlobal import directNotify
 from direct.showbase.PythonUtil import Queue, invertDictLossless
 from direct.showbase.PythonUtil import itype, serialNum, safeRepr
 from direct.showbase.Job import Job
-import types, weakref, random, __builtin__
+import types, weakref, gc, random, __builtin__
+
+def _createContainerLeak():
+    def leakContainer(task):
+        if not hasattr(simbase, 'leakContainer'):
+            simbase.leakContainer = []
+        simbase.leakContainer.append(1)
+        return task.cont
+    taskMgr.add(leakContainer, 'leakContainer-%s' % serialNum())
 
 class CheckContainers(Job):
     """
@@ -13,13 +21,18 @@ class CheckContainers(Job):
         self._leakDetector = leakDetector
         self.notify = self._leakDetector.notify
         self._index = index
+        ContainerLeakDetector.addPrivateId(self.__dict__)
+
+    def destroy(self):
+        ContainerLeakDetector.removePrivateId(self.__dict__)
+        Job.destroy(self)
 
     def getPriority(self):
         return Job.Priorities.Normal
     
     def run(self):
-        self._leakDetector._index2containerName2len[self._index] = {}
-        self._leakDetector.notify.debug(repr(self._leakDetector._id2ref))
+        self._leakDetector._index2containerId2len[self._index] = {}
+        #self._leakDetector.notify.debug(repr(self._leakDetector._id2ref))
         ids = self._leakDetector.getContainerIds()
         # record the current len of each container
         for id in ids:
@@ -28,47 +41,89 @@ class CheckContainers(Job):
                 container = self._leakDetector.getContainerById(id)
             except Exception, e:
                 # this container no longer exists
-                self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (name, e))
+                self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (
+                    self._leakDetector.getContainerNameById(id), e))
                 self._leakDetector.removeContainerById(id)
                 continue
             if container is None:
                 # this container no longer exists
-                self.notify.debug('container %s no longer exists; getContainerById returned None (%s)' % (name, e))
+                self.notify.debug('container %s no longer exists; getContainerById returned None' %
+                                  self._leakDetector.getContainerNameById(id))
+                self._leakDetector.removeContainerById(id)
+                continue
+            try:
+                cLen = len(container)
+            except Exception, e:
+                # this container no longer exists
+                self.notify.debug('%s is no longer a container, it is now %s (%s)' %
+                                  (self._leakDetector.getContainerNameById(id), safeRepr(container), e))
                 self._leakDetector.removeContainerById(id)
                 continue
-            cLen = len(container)
-            name = self._leakDetector.getContainerNameById(id)
-            self._leakDetector._index2containerName2len[self._index][name] = cLen
+            self._leakDetector._index2containerId2len[self._index][id] = cLen
         # compare the current len of each container to past lens
         if self._index > 0:
-            idx2name2len = self._leakDetector._index2containerName2len
-            for name in idx2name2len[self._index]:
+            idx2id2len = self._leakDetector._index2containerId2len
+            for id in idx2id2len[self._index]:
                 yield None
-                if name in idx2name2len[self._index-1]:
-                    diff = idx2name2len[self._index][name] - idx2name2len[self._index-1][name]
+                if id in idx2id2len[self._index-1]:
+                    diff = idx2id2len[self._index][id] - idx2id2len[self._index-1][id]
                     if diff > 0:
-                        if diff > idx2name2len[self._index-1][name]:
+                        if diff > idx2id2len[self._index-1][id]:
                             minutes = (self._leakDetector._index2delay[self._index] -
                                        self._leakDetector._index2delay[self._index-1]) / 60.
+                            name = self._leakDetector.getContainerNameById(id)
                             self.notify.warning('container %s grew > 200%% in %s minutes' % (name, minutes))
-                    if self._index > 3:
-                        diff2 = idx2name2len[self._index-1][name] - idx2name2len[self._index-2][name]
-                        diff3 = idx2name2len[self._index-2][name] - idx2name2len[self._index-3][name]
+                    if (self._index > 3 and
+                        id in idx2id2len[self._index-2] and
+                        id in idx2id2len[self._index-3]):
+                        diff2 = idx2id2len[self._index-1][id] - idx2id2len[self._index-2][id]
+                        diff3 = idx2id2len[self._index-2][id] - idx2id2len[self._index-3][id]
                         if self._index <= 5:
-                            msg = ('%s consistently increased in length over the last 3 periods (currently %s items)' %
-                                   (name, idx2name2len[self._index][name]))
-                            self.notify.warning(msg)
-                        else:
+                            if diff > 0 and diff2 > 0 and diff3 > 0:
+                                name = self._leakDetector.getContainerNameById(id)
+                                msg = ('%s consistently increased in length over the last 3 periods (currently %s items)' %
+                                       (name, idx2id2len[self._index][id]))
+                                self.notify.warning(msg)
+                        elif (id in idx2id2len[self._index-4] and
+                              id in idx2id2len[self._index-5]):
                             # if size has consistently increased over the last 5 checks, send out a warning
-                            diff4 = idx2name2len[self._index-3][name] - idx2name2len[self._index-4][name]
-                            diff5 = idx2name2len[self._index-4][name] - idx2name2len[self._index-5][name]
+                            diff4 = idx2id2len[self._index-3][id] - idx2id2len[self._index-4][id]
+                            diff5 = idx2id2len[self._index-4][id] - idx2id2len[self._index-5][id]
                             if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
+                                name = self._leakDetector.getContainerNameById(id)
                                 msg = ('%s consistently increased in length over the last 5 periods (currently %s items), notifying system' %
-                                       (name, idx2name2len[self._index][name]))
+                                       (name, idx2id2len[self._index][id]))
                                 self.notify.warning(msg)
                                 messenger.send(self._leakDetector.getLeakEvent(), [msg])
         yield Job.Done
 
+class PruneContainerRefs(Job):
+    """
+    Job to destroy any container refs that have Indirections that are holding references
+    to objects that should be garbage-collected
+    """
+    def __init__(self, name, leakDetector):
+        Job.__init__(self, name)
+        self._leakDetector = leakDetector
+        self.notify = self._leakDetector.notify
+        self._index = index
+        ContainerLeakDetector.addPrivateId(self.__dict__)
+
+    def destroy(self):
+        ContainerLeakDetector.removePrivateId(self.__dict__)
+        Job.destroy(self)
+
+    def getPriority(self):
+        return Job.Priorities.Normal
+    
+    def run(self):
+        ids = self._leakDetector._id2ref.keys()
+        for id in ids:
+            yield None
+            ref = self._leakDetector._id2ref[id]
+            ref.destroyIfGarbageDictKey()
+        yield Job.Done
+
 class NoDictKey:
     pass
 
@@ -76,13 +131,14 @@ class Indirection:
     # represents the indirection that brings you from a container to an element of the container
     # stored as a string to be used as part of an eval
     # each dictionary dereference is individually eval'd since the dict key might have been garbage-collected
-    class GarbageCollectedDictKey(Exception):
-        pass
 
     def __init__(self, evalStr=None, dictKey=NoDictKey):
         # if this is a dictionary lookup, pass dictKey instead of evalStr
         self.evalStr = evalStr
         self.dictKey = NoDictKey
+        # is the dictKey a weak reference?
+        self._isWeakRef = False
+        self._refCount = 0
         if dictKey is not NoDictKey:
             # if we can repr/eval the key, store it as an evalStr
             keyRepr = repr(dictKey)
@@ -100,34 +156,74 @@ class Indirection:
                 # eval/repr succeeded, store as an evalStr
                 self.evalStr = '[%s]' % keyRepr
             else:
-                # store a weakref to the key
-                self.dictKey = weakref.ref(dictKey)
+                try:
+                    # store a weakref to the key
+                    self.dictKey = weakref.ref(dictKey)
+                    self._isWeakRef = True
+                except TypeError, e:
+                    self.dictKey = dictKey
+                    self._isWeakRef = False
+
+    def destroy(self):
+        del self.dictKey
+
+    def acquire(self):
+        self._refCount += 1
+    def release(self):
+        self._refCount -= 1
+        if self._refCount == 0:
+            self.destroy()
 
     def isDictKey(self):
+        # is this an indirection through a dictionary?
         return self.dictKey is not NoDictKey
+    def isGarbageDictKey(self):
+        # are we holding a non-weak reference to an object that should be
+        # garbage-collected?
+        if self.isDictKey() and not self._isWeakRef:
+            referrers = gc.get_referrers(self.dictKey)
+            print referrers
+            import pdb;pdb.set_trace()
+
+    def _getNonWeakDictKey(self):
+        if not self._isWeakRef:
+            return self.dictKey
+        else:
+            key = self.dictKey()
+            if key is None:
+                return '<garbage-collected dict key>'
+            return key
 
     def dereferenceDictKey(self, parentDict):
-        key = self.dictKey()
-        if key is None:
-            raise Indirection.GarbageCollectedDictKey()
+        # look ourselves up in parentDict
+        key = self._getNonWeakDictKey()
+        # objects in __builtin__ will have parentDict==None
+        if parentDict is None:
+            return key
         return parentDict[key]
 
-    def getString(self, nextIndirection=None):
-        # return our contribution to the name of the object
+    def getString(self, prevIndirection=None, nextIndirection=None):
+        # return our contribution to the full name of an object
+        instanceDictStr = '.__dict__'
         if self.evalStr is not None:
-            # if we're an instance dict and the next indirection is not a dict key,
-            # skip over this one (obj.__dict__[keyName] == obj.keyName)
-            if nextIndirection is not None and self.evalStr == '.__dict__':
-                return ''
+            # if we're an instance dict, skip over this one (obj.__dict__[keyName] == obj.keyName)
+            if nextIndirection is not None and self.evalStr[-len(instanceDictStr):] == instanceDictStr:
+                return self.evalStr[:-len(instanceDictStr)]
+            # if the previous indirection was an instance dict, change our syntax from ['key'] to .key
+            if prevIndirection is not None and prevIndirection.evalStr is not None:
+                if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
+                    return '.%s' % self.evalStr[2:-2]
             return self.evalStr
 
         # we're stored as a dict key
         # this might not eval, but that's OK, we're not using this string to find
         # the object, we dereference the parent dict
-        key = self.dictKey()
-        if key is None:
-            return '<garbage-collected dict key>'
-        return safeRepr(key)
+        keyRepr = safeRepr(self._getNonWeakDictKey())
+        # if the previous indirection was an instance dict, change our syntax from ['key'] to .key
+        if prevIndirection is not None and prevIndirection.evalStr is not None:
+            if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
+                return '.%s' % keyRepr
+        return '[%s]' % keyRepr
 
     def __repr__(self):
         return self.getString()
@@ -140,21 +236,30 @@ class ContainerRef:
     """
     class FailedEval(Exception):
         pass
-    # whatever this is set to will be the default ContainerRef
-    BaseRef = None
 
-    def __init__(self, other=None, indirection=None):
+    def __init__(self, indirection, other=None):
         self._indirections = []
         # if no other passed in, try ContainerRef.BaseRef
-        if other is None:
-            other = ContainerRef.BaseRef
         if other is not None:
             for ind in other._indirections:
                 self.addIndirection(ind)
-        if indirection:
-            self.addIndirection(indirection)
+        self.addIndirection(indirection)
+
+    def destroy(self):
+        for indirection in self._indirections:
+            indirection.release()
+        del self._indirections
+
+    def destroyIfGarbageDictKey(self):
+        # if any of our indirections are holding onto objects that
+        # should be garbage-collected, destroy
+        for indirection in self._indirections:
+            if indirection.isGarbageDictKey():
+                self.destroy()
+                return
 
     def addIndirection(self, indirection):
+        indirection.acquire()
         self._indirections.append(indirection)
 
     def _getContainerByEval(self, evalStr):
@@ -177,8 +282,6 @@ class ContainerRef:
         #import pdb;pdb.set_trace()
         evalStr = ''
         curObj = None
-        curIndirection = None
-        nextIndirection = None
         for indirection in self._indirections:
             if not indirection.isDictKey():
                 # build up a string to be eval'd
@@ -195,15 +298,21 @@ class ContainerRef:
         
     def __repr__(self):
         str = ''
+        prevIndirection = None
         curIndirection = None
         nextIndirection = None
         for i in xrange(len(self._indirections)):
+            if i > 0:
+                prevIndirection = self._indirections[i-1]
+            else:
+                prevIndirection = None
             curIndirection = self._indirections[i]
             if i < len(self._indirections)-1:
                 nextIndirection = self._indirections[i+1]
             else:
                 nextIndirection = None
-            str += curIndirection.getString(nextIndirection=nextIndirection)
+            str += curIndirection.getString(prevIndirection=prevIndirection,
+                                            nextIndirection=nextIndirection)
         return str
 
 class ContainerLeakDetector(Job):
@@ -226,33 +335,39 @@ class ContainerLeakDetector(Job):
         if firstCheckDelay is None:
             firstCheckDelay = 60. * 15.
         self._nextCheckDelay = firstCheckDelay
-        self._index2containerName2len = {}
+        self._pruneTaskPeriod = config.GetFloat('leak-detector-prune-period', 60. * 30.)
+        self._index2containerId2len = {}
         self._index2delay = {}
         # set up our data structures
         self._id2ref = {}
 
         # set up the base/starting object
-        self._nameContainer(__builtin__.__dict__, ContainerRef(indirection=Indirection(evalStr='__builtin__.__dict__')))
+        self._baseObjRef = ContainerRef(Indirection(evalStr='__builtin__.__dict__'))
+        self._nameContainer(__builtin__.__dict__, self._baseObjRef)
         try:
             base
         except:
             pass
         else:
-            ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='base.__dict__'))
-            self._nameContainer(base.__dict__, ContainerRef.BaseRef)
+            self._baseObjRef = ContainerRef(Indirection(evalStr='base.__dict__'))
+            self._nameContainer(base.__dict__, self._baseObjRef)
         try:
             simbase
         except:
             pass
         else:
-            ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='simbase.__dict__'))
-            self._nameContainer(simbase.__dict__, ContainerRef.BaseRef)
+            self._baseObjRef = ContainerRef(Indirection(evalStr='simbase.__dict__'))
+            self._nameContainer(simbase.__dict__, self._baseObjRef)
+
+        if config.GetBool('leak-container', 0):
+            _createContainerLeak()
+
+        self._curObjRef = self._baseObjRef
 
-        self._curObjRef = ContainerRef()
         jobMgr.add(self)
         ContainerLeakDetector.PrivateIds.update(set([
             id(ContainerLeakDetector.PrivateIds),
-            id(self._id2ref),
+            id(self.__dict__),
             ]))
 
     def destroy(self):
@@ -260,14 +375,23 @@ class ContainerLeakDetector(Job):
             jobMgr.remove(self._checkContainersJob)
             self._checkContainersJob = None
         del self._id2ref
-        del self._index2containerName2len
+        del self._index2containerId2len
         del self._index2delay
 
     def getPriority(self):
         return self._priority
 
-    def getCheckTaskName(self):
+    @classmethod
+    def addPrivateId(cls, obj):
+        cls.PrivateIds.add(id(obj))
+    @classmethod
+    def removePrivateId(cls, obj):
+        cls.PrivateIds.remove(id(obj))
+
+    def _getCheckTaskName(self):
         return 'checkForLeakingContainers-%s' % self._serialNum
+    def _getPruneTaskName(self):
+        return 'pruneLeakingContainerRefs-%s' % self._serialNum
 
     def getLeakEvent(self):
         # passes description string as argument
@@ -285,7 +409,9 @@ class ContainerLeakDetector(Job):
 
     def run(self):
         taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks,
-                              self.getCheckTaskName())
+                              self._getCheckTaskName())
+        taskMgr.doMethodLater(self._pruneTaskPeriod, self._pruneContainerRefs,
+                              self._getPruneTaskName())
 
         while True:
             # yield up here instead of at the end, since we skip back to the
@@ -294,7 +420,7 @@ class ContainerLeakDetector(Job):
             #import pdb;pdb.set_trace()
             curObj = None
             if self._curObjRef is None:
-                self._curObjRef = random.choice(self._id2ref.values())
+                self._curObjRef = self._baseObjRef
             try:
                 curObj = self._curObjRef.getContainer()
             except:
@@ -308,26 +434,17 @@ class ContainerLeakDetector(Job):
                     del self._id2ref[_id]
                 self._curObjRef = self._id2ref[_id]
             #print '%s: %s, %s' % (id(curObj), type(curObj), self._id2ref[id(curObj)])
-            self.notify.debug('--> %s' % self._curObjRef)
+            #self.notify.debug('--> %s' % self._curObjRef)
 
             # keep a copy of this obj's eval str, it might not be in _id2ref
             curObjRef = self._curObjRef
             # if we hit a dead end, start over at a container we know about
             self._curObjRef = None
 
-            try:
-                if curObj.__class__.__name__ == 'method-wrapper':
-                    continue
-            except:
-                pass
-
-            if type(curObj) in (types.StringType, types.UnicodeType):
-                continue
-            
             if type(curObj) in (types.ModuleType, types.InstanceType):
                 child = curObj.__dict__
                 if not self._isDeadEnd(child):
-                    self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.__dict__'))
+                    self._curObjRef = ContainerRef(Indirection(evalStr='.__dict__'), curObjRef)
                     if self._isContainer(child):
                         self._nameContainer(child, self._curObjRef)
                 continue
@@ -345,12 +462,13 @@ class ContainerLeakDetector(Job):
                     except KeyError, e:
                         self.notify.warning('could not index into %s with key %s' % (curObjRef, key))
                         continue
-                    if not self._isDeadEnd(attr):
+                    if not self._isDeadEnd(attr, key):
                         if curObj is __builtin__.__dict__:
                             indirection=Indirection(evalStr=key)
+                            self._curObjRef = ContainerRef(indirection)
                         else:
                             indirection=Indirection(dictKey=key)
-                        self._curObjRef = ContainerRef(curObjRef, indirection=indirection)
+                            self._curObjRef = ContainerRef(indirection, curObjRef)
                         if self._isContainer(attr):
                             self._nameContainer(attr, self._curObjRef)
                 del key
@@ -379,7 +497,7 @@ class ContainerLeakDetector(Job):
                         random.shuffle(attrs)
                         for attr in attrs:
                             if not self._isDeadEnd(attr):
-                                self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='[%s]' % index))
+                                self._curObjRef = ContainerRef(Indirection(evalStr='[%s]' % index), curObjRef)
                                 if self._isContainer(attr):
                                     self._nameContainer(attr, self._curObjRef)
                             index += 1
@@ -401,8 +519,8 @@ class ContainerLeakDetector(Job):
                 random.shuffle(childNames)
                 for childName in childNames:
                     child = getattr(curObj, childName)
-                    if not self._isDeadEnd(child):
-                        self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.%s' % childName))
+                    if not self._isDeadEnd(child, childName):
+                        self._curObjRef = ContainerRef(Indirection(evalStr='.%s' % childName), curObjRef)
                         if self._isContainer(child):
                             self._nameContainer(child, self._curObjRef)
                 del childName
@@ -411,19 +529,31 @@ class ContainerLeakDetector(Job):
 
         yield Job.Done
         
-    def _isDeadEnd(self, obj):
+    def _isDeadEnd(self, obj, objName=None):
         if type(obj) in (types.BooleanType, types.BuiltinFunctionType,
                          types.BuiltinMethodType, types.ComplexType,
                          types.FloatType, types.IntType, types.LongType,
                          types.NoneType, types.NotImplementedType,
-                         types.TypeType, types.CodeType, types.FunctionType):
+                         types.TypeType, types.CodeType, types.FunctionType,
+                         types.StringType, types.UnicodeType):
+            return True
+        if id(obj) in self._id2ref:
             return True
         # if it's an internal object, ignore it
         if id(obj) in ContainerLeakDetector.PrivateIds:
             return True
-        if id(obj) in self._id2ref:
+        if objName in ('im_self', 'im_class'):
             return True
+        try:
+            className = obj.__class__.__name__
+        except:
+            pass
+        else:
+            # prevent infinite recursion in built-in containers related to methods
+            if className == 'method-wrapper':
+                return True
         return False
+            
 
     def _isContainer(self, obj):
         try:
@@ -433,9 +563,11 @@ class ContainerLeakDetector(Job):
         return True
 
     def _nameContainer(self, cont, objRef):
+        """
         if self.notify.getDebug():
             self.notify.debug('_nameContainer: %s' % objRef)
             #printStack()
+            """
         contId = id(cont)
         # if this container is new, or the objRef repr is shorter than what we already have,
         # put it in the table
@@ -443,11 +575,18 @@ class ContainerLeakDetector(Job):
             self._id2ref[contId] = objRef
 
     def _checkForLeaks(self, task=None):
-        self._index2delay[len(self._index2containerName2len)] = self._nextCheckDelay
+        self._index2delay[len(self._index2containerId2len)] = self._nextCheckDelay
         self._checkContainersJob = CheckContainers(
-            '%s-checkForLeaks' % self.getJobName(), self, len(self._index2containerName2len))
+            '%s-checkForLeaks' % self.getJobName(), self, len(self._index2containerId2len))
         jobMgr.add(self._checkContainersJob)
         
         self._nextCheckDelay *= 2
         taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks,
-                              self.getCheckTaskName())
+                              self._getCheckTaskName())
+        return task.done
+
+    def _pruneContainerRefs(self, task=None):
+        taskMgr.doMethodLater(self._pruneTaskPeriod, self._pruneContainerRefs,
+                              self._getPruneTaskName())
+        return task.done
+