mirror of https://github.com/scrapy/scrapy.git
107 lines
3.0 KiB
Python
107 lines
3.0 KiB
Python
"""
|
|
Scrapy extension for collecting scraping stats
|
|
"""
|
|
import pprint
|
|
|
|
from scrapy.xlib.pydispatch import dispatcher
|
|
|
|
from scrapy.signals import stats_spider_opened, stats_spider_closing, \
|
|
stats_spider_closed
|
|
from scrapy.utils.signal import send_catch_log
|
|
from scrapy import signals
|
|
from scrapy import log
|
|
from scrapy.conf import settings
|
|
|
|
class StatsCollector(object):
|
|
|
|
def __init__(self):
|
|
self._dump = settings.getbool('STATS_DUMP')
|
|
self._stats = {None: {}} # None is for global stats
|
|
dispatcher.connect(self._engine_stopped, signal=signals.engine_stopped)
|
|
|
|
def get_value(self, key, default=None, spider=None):
|
|
return self._stats[spider].get(key, default)
|
|
|
|
def get_stats(self, spider=None):
|
|
return self._stats[spider]
|
|
|
|
def set_value(self, key, value, spider=None):
|
|
self._stats[spider][key] = value
|
|
|
|
def set_stats(self, stats, spider=None):
|
|
self._stats[spider] = stats
|
|
|
|
def inc_value(self, key, count=1, start=0, spider=None):
|
|
d = self._stats[spider]
|
|
d[key] = d.setdefault(key, start) + count
|
|
|
|
def max_value(self, key, value, spider=None):
|
|
d = self._stats[spider]
|
|
d[key] = max(d.setdefault(key, value), value)
|
|
|
|
def min_value(self, key, value, spider=None):
|
|
d = self._stats[spider]
|
|
d[key] = min(d.setdefault(key, value), value)
|
|
|
|
def clear_stats(self, spider=None):
|
|
self._stats[spider].clear()
|
|
|
|
def iter_spider_stats(self):
|
|
return [x for x in self._stats.iteritems() if x[0]]
|
|
|
|
def open_spider(self, spider):
|
|
self._stats[spider] = {}
|
|
send_catch_log(stats_spider_opened, spider=spider)
|
|
|
|
def close_spider(self, spider, reason):
|
|
send_catch_log(stats_spider_closing, spider=spider, reason=reason)
|
|
stats = self._stats.pop(spider)
|
|
send_catch_log(stats_spider_closed, spider=spider, reason=reason, \
|
|
spider_stats=stats)
|
|
if self._dump:
|
|
log.msg("Dumping spider stats:\n" + pprint.pformat(stats), \
|
|
spider=spider)
|
|
self._persist_stats(stats, spider)
|
|
|
|
def _engine_stopped(self):
|
|
stats = self.get_stats()
|
|
if self._dump:
|
|
log.msg("Dumping global stats:\n" + pprint.pformat(stats))
|
|
self._persist_stats(stats, spider=None)
|
|
|
|
def _persist_stats(self, stats, spider=None):
|
|
pass
|
|
|
|
class MemoryStatsCollector(StatsCollector):
|
|
|
|
def __init__(self):
|
|
super(MemoryStatsCollector, self).__init__()
|
|
self.spider_stats = {}
|
|
|
|
def _persist_stats(self, stats, spider=None):
|
|
if spider is not None:
|
|
self.spider_stats[spider.name] = stats
|
|
|
|
|
|
class DummyStatsCollector(StatsCollector):
|
|
|
|
def get_value(self, key, default=None, spider=None):
|
|
return default
|
|
|
|
def set_value(self, key, value, spider=None):
|
|
pass
|
|
|
|
def set_stats(self, stats, spider=None):
|
|
pass
|
|
|
|
def inc_value(self, key, count=1, start=0, spider=None):
|
|
pass
|
|
|
|
def max_value(self, key, value, spider=None):
|
|
pass
|
|
|
|
def min_value(self, key, value, spider=None):
|
|
pass
|
|
|
|
|