RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added unittest for SpiderState extension

This commit is contained in:
Pablo Hoffman 2011-09-04 08:58:23 -03:00
parent e0ec239930
commit 17cc90e3fe
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/contrib/spiderstate.py
View File

@ -7,19 +7,20 @@ from scrapy.exceptions import NotConfigured
from scrapy.xlib.pydispatch import dispatcher
class SpiderState(object):
"""Stores and loads spider state"""
"""Store and load spider state during a scraping job"""
def __init__(self, jobdir):
self.statefn = os.path.join(jobdir, 'spider.state')
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
@classmethod
def from_crawler(cls, crawler):
jobdir = crawler.settings.get('JOBDIR')
if not jobdir:
raise NotConfigured
return cls(jobdir)
obj = cls(jobdir)
dispatcher.connect(obj.spider_closed, signal=signals.spider_closed)
dispatcher.connect(obj.spider_opened, signal=signals.spider_opened)
return obj
def spider_closed(self, spider):
with open(self.statefn, 'wb') as f: