RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added dumping of global scrapy stats at engine shutdown

This commit is contained in:
Pablo Hoffman 2009-07-16 10:48:10 -03:00
parent a4a1c4b130
commit c2f12327d6
2 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/ref/settings.rst
View File

@ -865,7 +865,9 @@ STATS_DUMP
Default: ``False``
Dump (to log) collected Scrapy stats when a domain is closed.
Dump (to log) domain-specific stats collected when a domain is closed, and all
global stats when the Scrapy process finishes (ie. when the engine is
shutdown).
.. setting:: STATS_ENABLED

8
scrapy/stats/collector/__init__.py
View File

@ -19,6 +19,7 @@ class StatsCollector(object):
dispatcher.connect(self.open_domain, signal=signals.domain_open)
dispatcher.connect(self.close_domain, signal=signals.domain_closed)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
def get_value(self, key, default=None, domain=None):
return self._stats[domain].get(key, default)
@ -49,10 +50,15 @@ class StatsCollector(object):
def close_domain(self, domain, reason):
signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason)
if self._dump:
log.msg("Dumping stats:\n" + pprint.pformat(self._stats[domain]), domain=domain)
log.msg("Dumping stats:\n" + pprint.pformat(self.get_stats(domain)), \
domain=domain)
del self._stats[domain]
signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason)
def engine_stopped(self):
if self._dump:
log.msg("Dumping global stats:\n" + pprint.pformat(self.get_stats()))
class MemoryStatsCollector(StatsCollector):