From 5fbc32c0156a5cf774bb9aa2be8a80f884ce3db2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 6 Jun 2011 03:12:40 -0300 Subject: [PATCH] call stats collector engine_stopped() after the engine is closed (to make sure all data from extensions has been collected), and added that method to documented api --- docs/topics/stats.rst | 4 ++++ scrapy/core/engine.py | 1 + scrapy/statscol.py | 3 +-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 072164e5b..69b08211f 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -185,6 +185,10 @@ class (which they all inherit from). for this spider can be accessed. This method is called automatically on the :signal:`spider_closed` signal. + .. method:: engine_stopped() + + Called after the engine is stopped, to dump or persist global stats. + Available Stats Collectors ========================== diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 994bd70cd..ec433ad0f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -260,3 +260,4 @@ class ExecutionEngine(object): @defer.inlineCallbacks def _finish_stopping_engine(self): yield send_catch_log_deferred(signal=signals.engine_stopped) + yield stats.engine_stopped() diff --git a/scrapy/statscol.py b/scrapy/statscol.py index c69510dd0..b4ecb2f05 100644 --- a/scrapy/statscol.py +++ b/scrapy/statscol.py @@ -17,7 +17,6 @@ class StatsCollector(object): def __init__(self): self._dump = settings.getbool('STATS_DUMP') self._stats = {None: {}} # None is for global stats - dispatcher.connect(self._engine_stopped, signal=signals.engine_stopped) def get_value(self, key, default=None, spider=None): return self._stats[spider].get(key, default) @@ -63,7 +62,7 @@ class StatsCollector(object): spider=spider) self._persist_stats(stats, spider) - def _engine_stopped(self): + def engine_stopped(self): stats = self.get_stats() if self._dump: log.msg("Dumping global stats:\n" + pprint.pformat(stats))