simplified MemoryDebugger extension to use stats for dumping memory debugging info

docs/topics/extensions.rst

@@ -256,11 +256,14 @@ Memory debugger extension
 
 .. class:: scrapy.contrib.memdebug.MemoryDebugger
 
-A memory debugger which collects some info about objects uncollected by the
-garbage collector and libxml2 memory leaks. To enable this extension, turn on
-the :setting:`MEMDEBUG_ENABLED` setting. The report will be printed to standard
-output. If the :setting:`MEMDEBUG_NOTIFY` setting contains a list of e-mails the
-report will also be sent to those addresses.
+An extension for debugging memory usage. It collects information about:
+
+* objects uncollected by the Python garbage collector
+* libxml2 memory leaks
+* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
+
+To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
+info will be stored in the stats.
 
 Close spider extension
 ~~~~~~~~~~~~~~~~~~~~~~

scrapy/contrib/memdebug.py

@@ -4,18 +4,15 @@ MemoryDebugger extension
 See documentation in docs/topics/extensions.rst
 """
 
-import os
 import gc
-import socket
 
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
-from scrapy.mail import MailSender
-from scrapy.utils.trackref import format_live_refs
 from scrapy.conf import settings
-from scrapy import log
+from scrapy.stats import stats
+from scrapy.utils.trackref import live_refs
 
 class MemoryDebugger(object):
 
@@ -28,9 +25,6 @@ class MemoryDebugger(object):
         if not settings.getbool('MEMDEBUG_ENABLED'):
             raise NotConfigured
 
-        self.mail = MailSender()
-        self.rcpts = settings.getlist('MEMDEBUG_NOTIFY')
-
         dispatcher.connect(self.engine_started, signals.engine_started)
         dispatcher.connect(self.engine_stopped, signals.engine_stopped)
 
@@ -39,32 +33,13 @@ class MemoryDebugger(object):
             self.libxml2.debugMemory(1)
 
     def engine_stopped(self):
-        figures = self.collect_figures()
-        report = self.create_report(figures)
-        self.log_or_send_report(report)
-
-    def collect_figures(self):
-        gc.collect()
-
-        figures = []
-        figures.append(("Objects in gc.garbage", len(gc.garbage), ""))
         if self.libxml2:
             self.libxml2.cleanupParser()
-            figures.append(("libxml2 memory leak", self.libxml2.debugMemory(1), "bytes"))
-        return figures
-
-    def create_report(self, figures):
-        s = ""
-        s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n"
-        for f in figures:
-            s += "%-30s : %d %s\n" % f
+            stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1))
+        gc.collect()
+        stats.set_value('memdebug/gc_garbage_count', len(gc.garbage))
         if settings.getbool('TRACK_REFS'):
-            s += os.linesep
-            s += format_live_refs()
-        return s
-
-    def log_or_send_report(self, report):
-        if self.rcpts:
-            self.mail.send(self.rcpts, "Scrapy Memory Debugger results at %s" % \
-                socket.gethostname(), report)
-        log.msg(report)
+            for cls, wdict in live_refs.iteritems():
+                if not wdict:
+                    continue
+                stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict))