diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py index dd61721ca..4f2567c3f 100644 --- a/scrapy/contrib/logstats.py +++ b/scrapy/contrib/logstats.py @@ -3,20 +3,13 @@ from twisted.internet import task from scrapy.exceptions import NotConfigured from scrapy import log, signals -class Slot(object): - - def __init__(self): - self.items = 0 - self.itemsprev = 0 - self.pages = 0 - self.pagesprev = 0 class LogStats(object): """Log basic scraping stats periodically""" - def __init__(self, interval=60.0): + def __init__(self, stats, interval=60.0): + self.stats = stats self.interval = interval - self.slots = {} self.multiplier = 60.0 / self.interval @classmethod @@ -24,40 +17,28 @@ class LogStats(object): interval = crawler.settings.getfloat('LOGSTATS_INTERVAL') if not interval: raise NotConfigured - o = cls(interval) - crawler.signals.connect(o.item_scraped, signal=signals.item_scraped) - crawler.signals.connect(o.response_received, signal=signals.response_received) + o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) - crawler.signals.connect(o.engine_started, signal=signals.engine_started) - crawler.signals.connect(o.engine_stopped, signal=signals.engine_stopped) return o - def item_scraped(self, spider): - self.slots[spider].items += 1 - - def response_received(self, spider): - self.slots[spider].pages += 1 - def spider_opened(self, spider): - self.slots[spider] = Slot() + self.pagesprev = 0 + self.itemsprev = 0 - def spider_closed(self, spider): - del self.slots[spider] + self.task = task.LoopingCall(self.log, spider) + self.task.start(self.interval) - def engine_started(self): - self.tsk = task.LoopingCall(self.log) - self.tsk.start(self.interval) + def log(self, spider): + items = self.stats.get_value('item_scraped_count', 0) + pages = self.stats.get_value('response_received_count', 0) + irate = (items - self.itemsprev) * self.multiplier + prate = (pages - self.pagesprev) * self.multiplier + self.pagesprev, self.itemsprev = pages, items + msg = "Crawled %d pages (at %d pages/min), scraped %d items (at %d items/min)" \ + % (pages, prate, items, irate) + log.msg(msg, spider=spider) - def log(self): - for spider, slot in self.slots.items(): - irate = (slot.items - slot.itemsprev) * self.multiplier - prate = (slot.pages - slot.pagesprev) * self.multiplier - slot.pagesprev, slot.itemsprev = slot.pages, slot.items - msg = "Crawled %d pages (at %d pages/min), scraped %d items (at %d items/min)" \ - % (slot.pages, prate, slot.items, irate) - log.msg(msg, spider=spider) - - def engine_stopped(self): - if self.tsk.running: - self.tsk.stop() + def spider_closed(self, spider, reason): + if self.task.running: + self.task.stop()