From 91dc46539fbcfd0df553d632676254d46d97f9e7 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 14 Jun 2011 00:50:05 -0300
Subject: [PATCH] added LogStats extension for periodically logging basic stats
 (like crawled pages and scraped items)

---
 docs/topics/extensions.rst          | 12 +++++-
 docs/topics/settings.rst            |  2 +
 scrapy/contrib/logstats.py          | 57 +++++++++++++++++++++++++++++
 scrapy/settings/default_settings.py |  3 ++
 4 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/contrib/logstats.py

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 5848471a2..b227fa493 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -178,10 +178,20 @@ Built-in extensions reference
 General purpose extensions
 --------------------------
 
+Log Stats extension
+~~~~~~~~~~~~~~~~~~~
+
+.. module:: scrapy.contrib.logstats
+   :synopsis: Basic stats logging
+
+.. class:: LogStats
+
+Log basic stats like crawled pages and scraped items.
+
 Core Stats extension
 ~~~~~~~~~~~~~~~~~~~~
 
-.. module:: scrapy.contrib.corestats.corestats
+.. module:: scrapy.contrib.corestats
    :synopsis: Core stats collection
 
 .. class:: CoreStats
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 515fe503c..ae6beab03 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -548,6 +548,8 @@ Default::
         'scrapy.contrib.closespider.CloseSpider': 0,
         'scrapy.contrib.feedexport.FeedExporter': 0,
         'scrapy.contrib.spidercontext.SpiderContext': 0,
+        'scrapy.contrib.throttle.AutoThrottle': 0,
+        'scrapy.contrib.logstats.LogStats': 0,
     }
 
 The list of available extensions. Keep in mind that some of them need to
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
new file mode 100644
index 000000000..1e4a11675
--- /dev/null
+++ b/scrapy/contrib/logstats.py
@@ -0,0 +1,57 @@
+from collections import defaultdict
+
+from twisted.internet import task
+
+from scrapy.xlib.pydispatch import dispatcher
+from scrapy.exceptions import NotConfigured
+from scrapy.conf import settings
+from scrapy import log, signals
+
+class Slot(object):
+
+    def __init__(self):
+        self.items = 0
+        self.itemsprev = 0
+        self.pages = 0
+        self.pagesprev = 0
+
+class LogStats(object):
+    """Log basic scraping stats periodically"""
+
+    def __init__(self):
+        self.interval = settings.getfloat('LOGSTATS_INTERVAL')
+        if not self.interval:
+            raise NotConfigured
+        self.slots = defaultdict(Slot)
+        self.multiplier = 60.0 / self.interval
+        dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
+        dispatcher.connect(self.response_received, signal=signals.response_received)
+        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
+        dispatcher.connect(self.engine_started, signal=signals.engine_started)
+        dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
+
+    def item_scraped(self, spider):
+        self.slots[spider].items += 1
+
+    def response_received(self, spider):
+        self.slots[spider].pages += 1
+
+    def spider_closed(self, spider):
+        del self.slots[spider]
+
+    def engine_started(self):
+        self.tsk = task.LoopingCall(self.log)
+        self.tsk.start(self.interval)
+
+    def log(self):
+        for spider, slot in self.slots.items():
+            irate = (slot.items - slot.itemsprev) * self.multiplier
+            prate = (slot.pages - slot.pagesprev) * self.multiplier
+            slot.pagesprev, slot.itemsprev = slot.pages, slot.items
+            msg = "Crawled %d pages (at %d pages/min), scraped %d items (at %d items/min)" \
+                % (slot.pages, prate, slot.items, irate)
+            log.msg(msg, spider=spider)
+
+    def engine_stopped(self):
+        if self.tsk.running:
+            self.tsk.stop()
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0ac7e9216..cc4bc2387 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -137,6 +137,7 @@ EXTENSIONS_BASE = {
     'scrapy.contrib.feedexport.FeedExporter': 0,
     'scrapy.contrib.spidercontext.SpiderContext': 0,
     'scrapy.contrib.throttle.AutoThrottle': 0,
+    'scrapy.contrib.logstats.LogStats': 0,
 }
 
 FEED_URI = None
@@ -182,6 +183,8 @@ LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
 
+LOGSTATS_INTERVAL = 60.0
+
 MAIL_DEBUG = False
 MAIL_HOST = 'localhost'
 MAIL_PORT = 25