From 0349bbf9d3691ad89a5e265db4220fb3e64324ff Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 15:25:15 +0100 Subject: [PATCH 1/2] Disable SpiderState extension if no JOBDIR set --- scrapy/extensions/spiderstate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 3799c7c66..2220cbd8f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -2,6 +2,7 @@ import os from six.moves import cPickle as pickle from scrapy import signals +from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir class SpiderState(object): @@ -12,7 +13,11 @@ class SpiderState(object): @classmethod def from_crawler(cls, crawler): - obj = cls(job_dir(crawler.settings)) + jobdir = job_dir(crawler.settings) + if not jobdir: + raise NotConfigured + + obj = cls(jobdir) crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed) crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj From 29695375d16ae20e3a97dc78bc12662996a9319b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Jan 2016 16:33:24 +0100 Subject: [PATCH 2/2] Add test for raised exception with SpiderState extension when no JOBDIR used --- tests/test_spiderstate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index d83015bd9..d1d6debec 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -4,6 +4,8 @@ from twisted.trial import unittest from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider +from scrapy.exceptions import NotConfigured +from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): @@ -34,3 +36,7 @@ class SpiderStateTest(unittest.TestCase): ss.spider_opened(spider) self.assertEqual(spider.state, {}) ss.spider_closed(spider) + + def test_not_configured(self): + crawler = get_crawler(Spider) + self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)