diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index b394b5e76..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -41,7 +41,7 @@ class Command(ScrapyCommand): log.msg("Could not find spider: %s" % opts.spider, log.ERROR) else: spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider()) + BaseSpider('default')) scrapymanager.crawl_request(request, spider) scrapymanager.start() diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py index 730ad0e8d..e2bf1a34b 100644 --- a/scrapy/contrib_exp/crawlspider/spider.py +++ b/scrapy/contrib_exp/crawlspider/spider.py @@ -15,9 +15,9 @@ class CrawlSpider(BaseSpider): request_processors = None rules = [] - def __init__(self): + def __init__(self, *a, **kw): """Initialize dispatcher""" - super(CrawlSpider, self).__init__() + super(CrawlSpider, self).__init__(*a, **kw) # auto follow start urls if self.start_urls: diff --git a/scrapy/shell.py b/scrapy/shell.py index 88d2b1710..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -55,7 +55,7 @@ class Shell(object): request = Request(url) spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider(), log_multiple=True) + BaseSpider('default'), log_multiple=True) print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index 615e920a6..c1ac2c1d4 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -52,7 +52,7 @@ class BaseSpider(object_ref): if not self.allowed_domains: self.allowed_domains = [] if not self.name: - self.name = 'default' + raise ValueError("%s must have a name" % type(self).__name__) # XXX: SEP-12 forward compatibility (remove for 0.10) self.domain_name = self.name diff --git a/scrapy/tests/test_contrib_exp_crawlspider_rules.py b/scrapy/tests/test_contrib_exp_crawlspider_rules.py index 0fbe52415..e04eb7b06 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_rules.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_rules.py @@ -109,7 +109,7 @@ class CompiledRuleInitializationTest(unittest.TestCase): class RulesTest(unittest.TestCase): def test_rules_manager_basic(self): - spider = BaseSpider() + spider = BaseSpider('foo') response1 = HtmlResponse('http://example.org') response2 = HtmlResponse('http://othersite.org') rulesman = RulesManager([], spider) @@ -134,7 +134,7 @@ class RulesTest(unittest.TestCase): self.failUnlessEqual(rule1.follow, True) def test_rules_manager_empty_rule(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') rulesman = RulesManager([Rule(follow=True)], spider) @@ -144,7 +144,7 @@ class RulesTest(unittest.TestCase): self.failUnless(isinstance(rule.matcher, BaseMatcher)) def test_rules_manager_default_matcher(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') callback = lambda x: None @@ -156,7 +156,7 @@ class RulesTest(unittest.TestCase): self.failUnless(isinstance(rule.matcher, UrlMatcher)) def test_rules_manager_matchers(self): - spider = BaseSpider() + spider = BaseSpider('foo') response1 = HtmlResponse('http://example.org') response2 = HtmlResponse('http://othersite.org') @@ -200,7 +200,7 @@ class RulesTest(unittest.TestCase): def test_rules_manager_callbacks(self): mycallback = lambda: True - spider = BaseSpider() + spider = BaseSpider('foo') spider.parse_item = lambda: True response1 = HtmlResponse('http://example.org') @@ -229,7 +229,7 @@ class RulesTest(unittest.TestCase): def test_rules_manager_callback_with_arguments(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') kwargs = {'a': 1} diff --git a/scrapy/tests/test_contrib_exp_crawlspider_spider.py b/scrapy/tests/test_contrib_exp_crawlspider_spider.py index 5e067508a..21f7f71e4 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_spider.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_spider.py @@ -60,7 +60,7 @@ class CrawlSpiderTest(unittest.TestCase): Spider.request_extractors = extractors Spider.request_processors = processors - return Spider() + return Spider('foo') def test_start_url_auto_rule(self): spider = self.spider_factory() diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index a23d6c6a4..ae92e16da 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -32,11 +32,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider()).addCallback(_test) + return download_file(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider()) + d = download_file(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -66,20 +66,20 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_timeout_download_from_spider(self): - spider = BaseSpider() + spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) d = download_http(request, spider) @@ -91,7 +91,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -99,9 +99,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -109,14 +109,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -152,7 +152,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -161,4 +161,4 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d1b4275e8..a04efd79d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -10,7 +10,7 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = CookiesMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index 1de53e85c..09dfdbaf8 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -21,7 +21,7 @@ class DecompressionMiddlewareTest(TestCase): def setUp(self): self.mw = DecompressionMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_known_compression_formats(self): for fmt in self.test_formats: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index cd1624eb4..805289787 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -9,7 +9,7 @@ from scrapy.spider import BaseSpider class TestDefaultHeadersMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() self.default_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/scrapy/tests/test_downloadermiddleware_httpauth.py index 79c815184..747ea9de1 100644 --- a/scrapy/tests/test_downloadermiddleware_httpauth.py +++ b/scrapy/tests/test_downloadermiddleware_httpauth.py @@ -18,7 +18,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): self.mw.default_useragent = 'default_useragent' - spider = TestSpider() + spider = TestSpider('foo') req = Request('http://scrapytest.org/') assert self.mw.process_request(req, spider) is None self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==') diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 9490fb40f..c8fb38d22 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -20,7 +20,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpCompressionMiddleware() def _getresponse(self, coding): diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py index dd9602661..7f6f7ee4e 100644 --- a/scrapy/tests/test_downloadermiddleware_httpproxy.py +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -8,7 +8,7 @@ from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.conf import settings -spider = BaseSpider() +spider = BaseSpider('foo') class TestDefaultHeadersMiddleware(TestCase): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index f9a93b910..8409fec6d 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -8,7 +8,7 @@ from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RedirectMiddleware() def test_priority_adjust(self): diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 101bb01c0..96e7cd297 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -10,7 +10,7 @@ from scrapy.http import Request, Response class RetryTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RetryMiddleware() self.mw.max_retry_times = 2 diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index d12f2bc63..338a63a23 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -9,7 +9,7 @@ from scrapy.conf import settings class UserAgentMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UserAgentMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index d11595a9d..fa9eef332 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -8,7 +8,7 @@ from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilt class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = RequestFingerprintDupeFilter() filter.open_spider(spider) @@ -28,7 +28,7 @@ class RequestFingerprintDupeFilterTest(unittest.TestCase): class NullDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = NullDupeFilter() filter.open_spider(spider) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index db59de2d5..1384c2237 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -10,7 +10,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = DuplicatesFilterMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw.open_spider(self.spider) def tearDown(self): diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 9a618e720..14aa78168 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -9,7 +9,6 @@ from scrapy.spider import BaseSpider from scrapy.contrib.spiders.init import InitSpider from scrapy.contrib.spiders.crawl import CrawlSpider from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter class BaseSpiderTest(unittest.TestCase): @@ -77,10 +76,8 @@ class BaseSpiderTest(unittest.TestCase): def test_spider_without_name(self): """Constructor arguments are assigned to spider attributes""" - spider = self.spider_class() - self.assertEqual(spider.name, 'default') - spider = self.spider_class(foo='bar') - self.assertEqual(spider.foo, 'bar') + self.assertRaises(ValueError, self.spider_class) + self.assertRaises(ValueError, self.spider_class, somearg='foo') class InitSpiderTest(BaseSpiderTest): diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index a0dfb492e..505b1a091 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpErrorMiddleware() self.req = Request('http://scrapytest.org') diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index cc40c8b20..861595524 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.spider.name = 'scrapytest.org' self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/scrapy/tests/test_spidermiddleware_referer.py index ec11b0489..467c301ba 100644 --- a/scrapy/tests/test_spidermiddleware_referer.py +++ b/scrapy/tests/test_spidermiddleware_referer.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.referer import RefererMiddleware class TestRefererMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RefererMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urlfilter.py b/scrapy/tests/test_spidermiddleware_urlfilter.py index 21f776f17..45621599a 100644 --- a/scrapy/tests/test_spidermiddleware_urlfilter.py +++ b/scrapy/tests/test_spidermiddleware_urlfilter.py @@ -9,7 +9,7 @@ from scrapy.utils.url import canonicalize_url class TestUrlFilterMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlFilterMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index de0e2eae0..cbd80d488 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -12,7 +12,7 @@ class TestUrlLengthMiddleware(TestCase): settings.disabled = False settings.overrides['URLLENGTH_LIMIT'] = 25 - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlLengthMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 612ee0eaa..786021898 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -9,7 +9,7 @@ from scrapy.stats.signals import stats_spider_opened, stats_spider_closing, \ class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_collector(self): stats = StatsCollector()