diff --git a/docs/news.rst b/docs/news.rst index 4b8440208..d9c95848e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -34,6 +34,7 @@ Scrapy changes: - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` +- removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled Scrapyd changes: diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 207cc36af..95a41f360 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -56,14 +56,9 @@ Debugging memory leaks with ``trackref`` memory leaks. It basically tracks the references to all live Requests, Responses, Item and Selector objects. -To activate the ``trackref`` module, enable the :setting:`TRACK_REFS` setting. -It only imposes a minor performance impact, so it should be OK to use it, even -in production environments. - -Once you have ``trackref`` enabled, you can enter the telnet console and inspect -how many objects (of the classes mentioned above) are currently alive using the -``prefs()`` function which is an alias to the -:func:`~scrapy.utils.trackref.print_live_refs` function:: +You can enter the telnet console and inspect how many objects (of the classes +mentioned above) are currently alive using the ``prefs()`` function which is an +alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:: telnet localhost 6023 diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py index d19786c13..18a746d31 100644 --- a/scrapy/contrib/debug.py +++ b/scrapy/contrib/debug.py @@ -19,7 +19,6 @@ class StackTraceDump(object): def __init__(self, crawler=None): self.crawler = crawler - self.dumprefs = crawler.settings.getbool('TRACK_REFS') try: signal.signal(signal.SIGUSR2, self.dump_stacktrace) signal.signal(signal.SIGQUIT, self.dump_stacktrace) @@ -34,7 +33,7 @@ class StackTraceDump(object): def dump_stacktrace(self, signum, frame): stackdumps = self._thread_stacks() enginestatus = format_engine_status(self.crawler.engine) - liverefs = format_live_refs() if self.dumprefs else "" + liverefs = format_live_refs() msg = "Dumping stack trace and engine status" \ "\n{0}\n{1}\n{2}".format(enginestatus, liverefs, stackdumps) log.msg(msg) diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index 8965a1ebd..359c7ec8a 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -12,20 +12,19 @@ from scrapy.utils.trackref import live_refs class MemoryDebugger(object): - def __init__(self, stats, trackrefs=False): + def __init__(self, stats): try: import libxml2 self.libxml2 = libxml2 except ImportError: self.libxml2 = None self.stats = stats - self.trackrefs = trackrefs @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured - o = cls(crawler.stats, crawler.settings.getbool('TRACK_REFS')) + o = cls(crawler.stats) crawler.signals.connect(o.engine_started, signals.engine_started) crawler.signals.connect(o.engine_stopped, signals.engine_stopped) return o @@ -40,8 +39,7 @@ class MemoryDebugger(object): self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) - if self.trackrefs: - for cls, wdict in live_refs.iteritems(): - if not wdict: - continue - self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) + for cls, wdict in live_refs.iteritems(): + if not wdict: + continue + self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index e28489ff3..8f3110f0a 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -2,8 +2,7 @@ references to live object instances. If you want live objects for a particular class to be tracked, you only have to -subclass form object_ref (instead of object). Also, remember to turn on -tracking by enabling the TRACK_REFS setting. +subclass form object_ref (instead of object). About performance: This library has a minimal performance impact when enabled, and no performance penalty at all when disabled (as object_ref becomes just an @@ -16,8 +15,6 @@ from time import time from operator import itemgetter from types import NoneType -from scrapy.conf import settings - live_refs = defaultdict(weakref.WeakKeyDictionary) class object_ref(object): @@ -31,12 +28,7 @@ class object_ref(object): live_refs[cls][obj] = time() return obj -if not settings.getbool('TRACK_REFS'): - object_ref = object - def format_live_refs(ignore=NoneType): - if object_ref is object: - return "The trackref module is disabled. Use TRACK_REFS setting to enable it." s = "Live References" + os.linesep + os.linesep now = time() for cls, wdict in live_refs.iteritems():