diff --git a/direct/src/showbase/ContainerLeakDetector.py b/direct/src/showbase/ContainerLeakDetector.py index 16dc336d08..d34463ca82 100755 --- a/direct/src/showbase/ContainerLeakDetector.py +++ b/direct/src/showbase/ContainerLeakDetector.py @@ -19,20 +19,25 @@ class CheckContainers(Job): def run(self): self._leakDetector._index2containerName2len[self._index] = {} - self._leakDetector.notify.debug(repr(self._leakDetector._id2pathStr)) - ids = self._leakDetector._id2pathStr.keys() + self._leakDetector.notify.debug(repr(self._leakDetector._id2ref)) + ids = self._leakDetector.getContainerIds() # record the current len of each container for id in ids: yield None - name = self._leakDetector._id2pathStr[id] try: - container = eval(name) + container = self._leakDetector.getContainerById(id) except Exception, e: # this container no longer exists - self.notify.debug('container %s no longer exists; caught exception in eval (%s)' % (name, e)) - del self._leakDetector._id2pathStr[id] + self.notify.debug('container %s no longer exists; caught exception in getContainerById (%s)' % (name, e)) + self._leakDetector.removeContainerById(id) + continue + if container is None: + # this container no longer exists + self.notify.debug('container %s no longer exists; getContainerById returned None (%s)' % (name, e)) + self._leakDetector.removeContainerById(id) continue cLen = len(container) + name = self._leakDetector.getContainerNameById(id) self._leakDetector._index2containerName2len[self._index][name] = cLen # compare the current len of each container to past lens if self._index > 0: @@ -64,6 +69,143 @@ class CheckContainers(Job): messenger.send(self._leakDetector.getLeakEvent(), [msg]) yield Job.Done +class NoDictKey: + pass + +class Indirection: + # represents the indirection that brings you from a container to an element of the container + # stored as a string to be used as part of an eval + # each dictionary dereference is individually eval'd since the dict key might have been garbage-collected + class GarbageCollectedDictKey(Exception): + pass + + def __init__(self, evalStr=None, dictKey=NoDictKey): + # if this is a dictionary lookup, pass dictKey instead of evalStr + self.evalStr = evalStr + self.dictKey = NoDictKey + if dictKey is not NoDictKey: + # if we can repr/eval the key, store it as an evalStr + keyRepr = repr(dictKey) + useEval = False + try: + keyEval = eval(keyRepr) + useEval = True + except: + pass + if useEval: + # check to make sure the eval succeeded + if hash(keyEval) != hash(dictKey): + useEval = False + if useEval: + # eval/repr succeeded, store as an evalStr + self.evalStr = '[%s]' % keyRepr + else: + # store a weakref to the key + self.dictKey = weakref.ref(dictKey) + + def isDictKey(self): + return self.dictKey is not NoDictKey + + def dereferenceDictKey(self, parentDict): + key = self.dictKey() + if key is None: + raise Indirection.GarbageCollectedDictKey() + return parentDict[key] + + def getString(self, nextIndirection=None): + # return our contribution to the name of the object + if self.evalStr is not None: + # if we're an instance dict and the next indirection is not a dict key, + # skip over this one (obj.__dict__[keyName] == obj.keyName) + if nextIndirection is not None and self.evalStr == '.__dict__': + return '' + return self.evalStr + + # we're stored as a dict key + # this might not eval, but that's OK, we're not using this string to find + # the object, we dereference the parent dict + key = self.dictKey() + if key is None: + return '' + return safeRepr(key) + + def __repr__(self): + return self.getString() + +class ContainerRef: + """ + stores a reference to a container in a way that does not prevent garbage + collection of the container + stored as a series of 'indirections' (obj.foo -> '.foo', dict[key] -> '[key]', etc.) + """ + class FailedEval(Exception): + pass + # whatever this is set to will be the default ContainerRef + BaseRef = None + + def __init__(self, other=None, indirection=None): + self._indirections = [] + # if no other passed in, try ContainerRef.BaseRef + if other is None: + other = ContainerRef.BaseRef + if other is not None: + for ind in other._indirections: + self.addIndirection(ind) + if indirection: + self.addIndirection(indirection) + + def addIndirection(self, indirection): + self._indirections.append(indirection) + + def _getContainerByEval(self, evalStr): + try: + container = eval(evalStr) + except NameError, ne: + return None + return container + + def _evalWithObj(self, evalStr, curObj=None): + # eval an evalStr, optionally based off of an existing object + if curObj is not None: + # eval('curObj.foo.bar.someDict') + evalStr = 'curObj%s' % evalStr + return self._getContainerByEval(evalStr) + + def getContainer(self): + # try to get a handle on the container by eval'ing and looking things + # up in dictionaries, depending on the type of each indirection + #import pdb;pdb.set_trace() + evalStr = '' + curObj = None + curIndirection = None + nextIndirection = None + for indirection in self._indirections: + if not indirection.isDictKey(): + # build up a string to be eval'd + evalStr += indirection.getString() + else: + curObj = self._evalWithObj(evalStr, curObj) + if curObj is None: + raise FailedEval(evalStr) + # try to look up this key in the curObj dictionary + curObj = indirection.dereferenceDictKey(curObj) + evalStr = '' + + return self._evalWithObj(evalStr, curObj) + + def __repr__(self): + str = '' + curIndirection = None + nextIndirection = None + for i in xrange(len(self._indirections)): + curIndirection = self._indirections[i] + if i < len(self._indirections)-1: + nextIndirection = self._indirections[i+1] + else: + nextIndirection = None + str += curIndirection.getString(nextIndirection=nextIndirection) + return str + class ContainerLeakDetector(Job): """ Low-priority Python object-graph walker that looks for leaking containers. @@ -87,19 +229,37 @@ class ContainerLeakDetector(Job): self._index2containerName2len = {} self._index2delay = {} # set up our data structures - self._id2pathStr = {} - self._curObjPathStr = '__builtin__.__dict__' + self._id2ref = {} + + # set up the base/starting object + self._nameContainer(__builtin__.__dict__, ContainerRef(indirection=Indirection(evalStr='__builtin__.__dict__'))) + try: + base + except: + pass + else: + ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='base.__dict__')) + self._nameContainer(base.__dict__, ContainerRef.BaseRef) + try: + simbase + except: + pass + else: + ContainerRef.BaseRef = ContainerRef(indirection=Indirection(evalStr='simbase.__dict__')) + self._nameContainer(simbase.__dict__, ContainerRef.BaseRef) + + self._curObjRef = ContainerRef() jobMgr.add(self) ContainerLeakDetector.PrivateIds.update(set([ id(ContainerLeakDetector.PrivateIds), - id(self._id2pathStr), + id(self._id2ref), ])) def destroy(self): if self._checkContainersJob is not None: jobMgr.remove(self._checkContainersJob) self._checkContainersJob = None - del self._id2pathStr + del self._id2ref del self._index2containerName2len del self._index2delay @@ -113,30 +273,17 @@ class ContainerLeakDetector(Job): # passes description string as argument return 'containerLeakDetected-%s' % self._serialNum - def _getContainerByEval(self, evalStr): - try: - container = eval(evalStr) - except NameError, ne: - return None - return container + def getContainerIds(self): + return self._id2ref.keys() + + def getContainerById(self, id): + return self._id2ref[id].getContainer() + def getContainerNameById(self, id): + return repr(self._id2ref[id]) + def removeContainerById(self, id): + del self._id2ref[id] def run(self): - # push on a few things that we want to give priority - # for the sake of the variable-name printouts - self._nameContainer(__builtin__.__dict__, '__builtin__.__dict__') - try: - base - except: - pass - else: - self._nameContainer(base.__dict__, 'base.__dict__') - try: - simbase - except: - pass - else: - self._nameContainer(simbase.__dict__, 'simbase.__dict__') - taskMgr.doMethodLater(self._nextCheckDelay, self._checkForLeaks, self.getCheckTaskName()) @@ -146,24 +293,27 @@ class ContainerLeakDetector(Job): yield None #import pdb;pdb.set_trace() curObj = None - curObj = self._getContainerByEval(self._curObjPathStr) - if curObj is None: - self.notify.debug('lost current container: %s' % self._curObjPathStr) - while len(self._id2pathStr): - _id = random.choice(self._id2pathStr.keys()) - curObj = self._getContainerByEval(self._id2pathStr[_id]) + if self._curObjRef is None: + self._curObjRef = random.choice(self._id2ref.values()) + try: + curObj = self._curObjRef.getContainer() + except: + self.notify.debug('lost current container: %s' % self._curObjRef) + while len(self._id2ref): + _id = random.choice(self._id2ref.keys()) + curObj = self.getContainerById(_id) if curObj is not None: break # container is no longer valid - del self._id2pathStr[_id] - self._curObjPathStr = self._id2pathStr[_id] - #print '%s: %s, %s' % (id(curObj), type(curObj), self._id2pathStr[id(curObj)]) - self.notify.debug('--> %s' % self._curObjPathStr) + del self._id2ref[_id] + self._curObjRef = self._id2ref[_id] + #print '%s: %s, %s' % (id(curObj), type(curObj), self._id2ref[id(curObj)]) + self.notify.debug('--> %s' % self._curObjRef) - # keep a copy of this obj's eval str, it might not be in _id2pathStr - curObjPathStr = self._curObjPathStr - # if we hit a dead end, go back to __builtin__ - self._curObjPathStr = '__builtin__' + # keep a copy of this obj's eval str, it might not be in _id2ref + curObjRef = self._curObjRef + # if we hit a dead end, start over at a container we know about + self._curObjRef = None try: if curObj.__class__.__name__ == 'method-wrapper': @@ -177,9 +327,9 @@ class ContainerLeakDetector(Job): if type(curObj) in (types.ModuleType, types.InstanceType): child = curObj.__dict__ if not self._isDeadEnd(child): - self._curObjPathStr = curObjPathStr + '.__dict__' + self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.__dict__')) if self._isContainer(child): - self._nameContainer(child, self._curObjPathStr) + self._nameContainer(child, self._curObjRef) continue if type(curObj) is types.DictType: @@ -193,26 +343,16 @@ class ContainerLeakDetector(Job): try: attr = curObj[key] except KeyError, e: - self.notify.warning('could not index into %s with key %s' % (curObjPathStr, - key)) + self.notify.warning('could not index into %s with key %s' % (curObjRef, key)) continue if not self._isDeadEnd(attr): - if curObj is __builtin__: - self._curObjPathStr = PathStr(key) - if key == '__doc__': - import pdb;pdb.set_trace() - if self._isContainer(attr): - self._nameContainer(attr, PathStr(key)) + if curObj is __builtin__.__dict__: + indirection=Indirection(evalStr=key) else: - # if the parent dictionary is an instance dictionary, remove the __dict__ - # and use the . operator - dLen = len('__dict__') - if len(self._curObjPathStr) >= dLen and self._curObjPathStr[-dLen:] == '__dict__': - self._curObjPathStr = curObjPathStr[:-dLen] + '.%s' % safeRepr(key) - else: - self._curObjPathStr = curObjPathStr + '[%s]' % safeRepr(key) - if self._isContainer(attr): - self._nameContainer(attr, self._curObjPathStr) + indirection=Indirection(dictKey=key) + self._curObjRef = ContainerRef(curObjRef, indirection=indirection) + if self._isContainer(attr): + self._nameContainer(attr, self._curObjRef) del key del attr continue @@ -239,9 +379,9 @@ class ContainerLeakDetector(Job): random.shuffle(attrs) for attr in attrs: if not self._isDeadEnd(attr): - self._curObjPathStr = curObjPathStr + '[%s]' % index + self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='[%s]' % index)) if self._isContainer(attr): - self._nameContainer(attr, self._curObjPathStr) + self._nameContainer(attr, self._curObjRef) index += 1 del attr except StopIteration, e: @@ -262,9 +402,9 @@ class ContainerLeakDetector(Job): for childName in childNames: child = getattr(curObj, childName) if not self._isDeadEnd(child): - self._curObjPathStr = curObjPathStr + '.%s' % childName + self._curObjRef = ContainerRef(curObjRef, indirection=Indirection(evalStr='.%s' % childName)) if self._isContainer(child): - self._nameContainer(child, self._curObjPathStr) + self._nameContainer(child, self._curObjRef) del childName del child continue @@ -281,6 +421,8 @@ class ContainerLeakDetector(Job): # if it's an internal object, ignore it if id(obj) in ContainerLeakDetector.PrivateIds: return True + if id(obj) in self._id2ref: + return True return False def _isContainer(self, obj): @@ -290,15 +432,15 @@ class ContainerLeakDetector(Job): return False return True - def _nameContainer(self, cont, pathStr): + def _nameContainer(self, cont, objRef): if self.notify.getDebug(): - self.notify.debug('_nameContainer: %s' % pathStr) - printStack() + self.notify.debug('_nameContainer: %s' % objRef) + #printStack() contId = id(cont) - # if this container is new, or the pathStr is shorter than what we already have, + # if this container is new, or the objRef repr is shorter than what we already have, # put it in the table - if contId not in self._id2pathStr or len(pathStr) < len(self._id2pathStr[contId]): - self._id2pathStr[contId] = pathStr + if contId not in self._id2ref or len(repr(objRef)) < len(repr(self._id2ref[contId])): + self._id2ref[contId] = objRef def _checkForLeaks(self, task=None): self._index2delay[len(self._index2containerName2len)] = self._nextCheckDelay