RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simplified CoreStats extension

This commit is contained in:
Pablo Hoffman 2011-06-05 19:57:38 -03:00
parent 61cc95df7c
commit 3d823d6f45
1 changed files with 2 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapy/contrib/corestats.py
View File

@ -1,26 +1,16 @@
"""
Scrapy extension for collecting scraping stats
Extension for collecting core stats like items scraped and start/finish times
"""
import os
import getpass
import socket
import datetime
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.stats import stats
from scrapy.conf import settings
class CoreStats(object):
"""Scrapy core stats collector"""
def __init__(self):
stats.set_value('envinfo/user', getpass.getuser())
stats.set_value('envinfo/host', socket.gethostname())
stats.set_value('envinfo/logfile', settings['LOG_FILE'])
stats.set_value('envinfo/pid', os.getpid())
dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened)
dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing)
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
@ -28,19 +18,15 @@ class CoreStats(object):
def stats_spider_opened(self, spider):
stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
stats.set_value('envinfo/host', stats.get_value('envinfo/host'), spider=spider)
def stats_spider_closing(self, spider, reason):
stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, spider=spider)
stats.inc_value('spiders_closed_count/%s' % reason)
stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):
stats.inc_value('item_scraped_count', spider=spider)
stats.inc_value('item_scraped_count')
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
stats.inc_value('item_dropped_count', spider=spider)
stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
stats.inc_value('item_dropped_count')