From 3d823d6f4525518bce26ee7550940dc669ef3108 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 5 Jun 2011 19:57:38 -0300 Subject: [PATCH] simplified CoreStats extension --- scrapy/contrib/corestats.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py index 7ab080735..da25d6a97 100644 --- a/scrapy/contrib/corestats.py +++ b/scrapy/contrib/corestats.py @@ -1,26 +1,16 @@ """ -Scrapy extension for collecting scraping stats +Extension for collecting core stats like items scraped and start/finish times """ -import os -import getpass -import socket import datetime from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.stats import stats -from scrapy.conf import settings class CoreStats(object): - """Scrapy core stats collector""" def __init__(self): - stats.set_value('envinfo/user', getpass.getuser()) - stats.set_value('envinfo/host', socket.gethostname()) - stats.set_value('envinfo/logfile', settings['LOG_FILE']) - stats.set_value('envinfo/pid', os.getpid()) - dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened) dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing) dispatcher.connect(self.item_scraped, signal=signals.item_scraped) @@ -28,19 +18,15 @@ class CoreStats(object): def stats_spider_opened(self, spider): stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) - stats.set_value('envinfo/host', stats.get_value('envinfo/host'), spider=spider) def stats_spider_closing(self, spider, reason): stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) - stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, spider=spider) - stats.inc_value('spiders_closed_count/%s' % reason) + stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): stats.inc_value('item_scraped_count', spider=spider) - stats.inc_value('item_scraped_count') def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ stats.inc_value('item_dropped_count', spider=spider) stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) - stats.inc_value('item_dropped_count')