Automated merge with http://hg.scrapy.org/users/ismael/scrapy-newitem/

docs/ref/settings.rst

@@ -865,7 +865,9 @@ STATS_DUMP
 
 Default: ``False``
 
-Dump (to log) collected Scrapy stats when a domain is closed.
+Dump (to log) domain-specific stats collected when a domain is closed, and all
+global stats when the Scrapy process finishes (ie. when the engine is
+shutdown).
 
 .. setting:: STATS_ENABLED
 

scrapy/contrib/statsmailer.py

@@ -4,14 +4,11 @@ StatsMailer extension sends an email when a domain finishes scraping.
 Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
 """
 
-import pprint
-
 from scrapy.xlib.pydispatch import dispatcher
 
-from scrapy.stats import stats
+from scrapy.stats import stats, signals
 from scrapy.mail import MailSender
 from scrapy.conf import settings
-from scrapy.core import signals
 from scrapy.core.exceptions import NotConfigured
 
 class StatsMailer(object):
@@ -20,9 +17,12 @@ class StatsMailer(object):
         self.recipients = settings.getlist("STATSMAILER_RCPTS")
         if not self.recipients:
             raise NotConfigured
-        dispatcher.connect(self.send_stats, signal=signals.domain_closed)
+        dispatcher.connect(self.send_stats, signal=signals.stats_domain_closing)
         
     def send_stats(self, domain):
         mail = MailSender()
-        body = pprint.pformat(stats[domain])
+        body = "Global stats\n\n"
+        body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items())
+        body += "\n\n%s stats\n\n" % domain
+        body += "\n".join("%-50s : %s" % i for i in stats.get_stats(domain).items())
         mail.send(self.recipients, "Scrapy stats for: %s" % domain, body)

scrapy/stats/collector/__init__.py

@@ -19,6 +19,7 @@ class StatsCollector(object):
 
         dispatcher.connect(self.open_domain, signal=signals.domain_open)
         dispatcher.connect(self.close_domain, signal=signals.domain_closed)
+        dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
 
     def get_value(self, key, default=None, domain=None):
         return self._stats[domain].get(key, default)
@@ -49,10 +50,15 @@ class StatsCollector(object):
     def close_domain(self, domain, reason):
         signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason)
         if self._dump:
-            log.msg("Dumping stats:\n" + pprint.pformat(self._stats[domain]), domain=domain)
+            log.msg("Dumping stats:\n" + pprint.pformat(self.get_stats(domain)), \
+                domain=domain)
         del self._stats[domain]
         signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason)
 
+    def engine_stopped(self):
+        if self._dump:
+            log.msg("Dumping global stats:\n" + pprint.pformat(self.get_stats()))
+
 
 class MemoryStatsCollector(StatsCollector):