diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py index b8bdd9cfc..d2cb4193a 100644 --- a/scrapy/contrib/spiderstate.py +++ b/scrapy/contrib/spiderstate.py @@ -9,26 +9,28 @@ from scrapy.xlib.pydispatch import dispatcher class SpiderState(object): """Store and load spider state during a scraping job""" - def __init__(self, jobdir): - self.statefn = os.path.join(jobdir, 'spider.state') + def __init__(self, jobdir=None): + self.jobdir = jobdir @classmethod def from_crawler(cls, crawler): - jobdir = crawler.settings.get('JOBDIR') - if not jobdir: - raise NotConfigured - obj = cls(jobdir) + obj = cls(crawler.settings.get('JOBDIR')) dispatcher.connect(obj.spider_closed, signal=signals.spider_closed) dispatcher.connect(obj.spider_opened, signal=signals.spider_opened) return obj def spider_closed(self, spider): - with open(self.statefn, 'wb') as f: - pickle.dump(spider.state, f, protocol=2) + if self.jobdir: + with open(self.statefn, 'wb') as f: + pickle.dump(spider.state, f, protocol=2) def spider_opened(self, spider): - if os.path.exists(self.statefn): + if self.jobdir and os.path.exists(self.statefn): with open(self.statefn) as f: spider.state = pickle.load(f) else: spider.state = {} + + @property + def statefn(self): + return os.path.join(self.jobdir, 'spider.state') diff --git a/scrapy/tests/test_contrib_spiderstate.py b/scrapy/tests/test_contrib_spiderstate.py index e340b489a..5a438917c 100644 --- a/scrapy/tests/test_contrib_spiderstate.py +++ b/scrapy/tests/test_contrib_spiderstate.py @@ -25,3 +25,12 @@ class SpiderStateTest(unittest.TestCase): ss2.spider_opened(spider2) self.assertEqual(spider.state, {'one': 1, 'dt': dt}) ss2.spider_closed(spider2) + + def test_state_attribute(self): + # state attribute must be present if jobdir is not set, to provide a + # consistent interface + spider = BaseSpider(name='default') + ss = SpiderState() + ss.spider_opened(spider) + self.assertEqual(spider.state, {}) + ss.spider_closed(spider)