diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 316e54937..e5fcc7bfe 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -865,7 +865,9 @@ STATS_DUMP Default: ``False`` -Dump (to log) collected Scrapy stats when a domain is closed. +Dump (to log) domain-specific stats collected when a domain is closed, and all +global stats when the Scrapy process finishes (ie. when the engine is +shutdown). .. setting:: STATS_ENABLED diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index c876714f6..ced8d6918 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -4,14 +4,11 @@ StatsMailer extension sends an email when a domain finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -import pprint - from scrapy.xlib.pydispatch import dispatcher -from scrapy.stats import stats +from scrapy.stats import stats, signals from scrapy.mail import MailSender from scrapy.conf import settings -from scrapy.core import signals from scrapy.core.exceptions import NotConfigured class StatsMailer(object): @@ -20,9 +17,12 @@ class StatsMailer(object): self.recipients = settings.getlist("STATSMAILER_RCPTS") if not self.recipients: raise NotConfigured - dispatcher.connect(self.send_stats, signal=signals.domain_closed) + dispatcher.connect(self.send_stats, signal=signals.stats_domain_closing) def send_stats(self, domain): mail = MailSender() - body = pprint.pformat(stats[domain]) + body = "Global stats\n\n" + body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) + body += "\n\n%s stats\n\n" % domain + body += "\n".join("%-50s : %s" % i for i in stats.get_stats(domain).items()) mail.send(self.recipients, "Scrapy stats for: %s" % domain, body) diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index bf1aa9e9d..44e52de39 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -19,6 +19,7 @@ class StatsCollector(object): dispatcher.connect(self.open_domain, signal=signals.domain_open) dispatcher.connect(self.close_domain, signal=signals.domain_closed) + dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) def get_value(self, key, default=None, domain=None): return self._stats[domain].get(key, default) @@ -49,10 +50,15 @@ class StatsCollector(object): def close_domain(self, domain, reason): signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason) if self._dump: - log.msg("Dumping stats:\n" + pprint.pformat(self._stats[domain]), domain=domain) + log.msg("Dumping stats:\n" + pprint.pformat(self.get_stats(domain)), \ + domain=domain) del self._stats[domain] signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason) + def engine_stopped(self): + if self._dump: + log.msg("Dumping global stats:\n" + pprint.pformat(self.get_stats())) + class MemoryStatsCollector(StatsCollector):