diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index e0a12ce21..316e54937 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -858,15 +858,14 @@ Default: ``'scrapy.stats.collector.MemoryStatsCollector'`` The class to use for collecting stats (must implement the Stats Collector API, or subclass the StatsCollector class). -.. setting:: STATS_DEBUG +.. setting:: STATS_DUMP -STATS_DEBUG ------------ +STATS_DUMP +---------- Default: ``False`` -Enable debugging mode for Scrapy stats. This logs the stats when a domain is -closed. +Dump (to log) collected Scrapy stats when a domain is closed. .. setting:: STATS_ENABLED diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index a0729e6d1..3622c1136 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -7,10 +7,10 @@ Stats Collector Overview ======== -Scrapy provides a convenient facility for collecting stats in the form of +Scrapy provides a convenient service for collecting stats in the form of key/values, both globally and per spider/domain. It's called the Stats Collector, and it's a singleton which can be imported and used quickly, as -illustrated by the example in the :ref:`topics-stats-usecases` section below. +illustrated by the examples in the :ref:`topics-stats-usecases` section below. The stats collection is enabled by default but can be disabled through the :setting:`STATS_ENABLED` setting. diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index b4592453e..a15b907ac 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -186,7 +186,7 @@ SPIDER_MIDDLEWARES_BASE = { STATS_CLASS = 'scrapy.stats.collector.MemoryStatsCollector' STATS_ENABLED = True -STATS_DEBUG = False +STATS_DUMP = False STATS_SDB_DOMAIN = 'scrapy_stats' STATS_SDB_ASYNC = False diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index 7d83d36b5..74fbcfc5d 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -14,7 +14,6 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.core.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.stats import stats from scrapy.extension import extensions from scrapy.conf import settings @@ -58,10 +57,6 @@ class MemoryDebugger(object): s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n" for f in figures: s += "%-30s : %s %s\n" % f - s += "\n" - if stats: - s += "SCRAPING STATS --------------------------------------------------\n\n" - s += pprint.pformat(stats) return s def print_or_send_report(self, report): diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index f3d86fb8b..5a9207dc7 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -17,7 +17,6 @@ from scrapy.core.manager import scrapymanager from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.stats import stats from scrapy.conf import settings class MemoryUsage(object): @@ -120,9 +119,4 @@ class MemoryUsage(object): s += "\r\n" s += scrapyengine.getstatus() s += "\r\n" - - if stats: - s += "SCRAPING STATS ------------------------------------------------------ \r\n" - s += "\r\n" - s += pprint.pformat(stats) self.mail.send(rcpts, subject, s) diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index 5402375bc..bf1aa9e9d 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -14,7 +14,7 @@ from scrapy.conf import settings class StatsCollector(object): def __init__(self): - self.debug = settings.getbool('STATS_DEBUG') + self._dump = settings.getbool('STATS_DUMP') self._stats = {None: {}} # None is for global stats dispatcher.connect(self.open_domain, signal=signals.domain_open) @@ -48,8 +48,8 @@ class StatsCollector(object): def close_domain(self, domain, reason): signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason) - if self.debug: - log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG) + if self._dump: + log.msg("Dumping stats:\n" + pprint.pformat(self._stats[domain]), domain=domain) del self._stats[domain] signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason)