made Spider name required again (do not default)

scrapy/command/commands/fetch.py

@@ -41,7 +41,7 @@ class Command(ScrapyCommand):
                 log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
         else:
             spider = scrapymanager._create_spider_for_request(request, \
-                BaseSpider())
+                BaseSpider('default'))
 
         scrapymanager.crawl_request(request, spider)
         scrapymanager.start()

scrapy/contrib_exp/crawlspider/spider.py

@@ -15,9 +15,9 @@ class CrawlSpider(BaseSpider):
     request_processors = None
     rules = []
 
-    def __init__(self):
+    def __init__(self, *a, **kw):
         """Initialize dispatcher"""
-        super(CrawlSpider, self).__init__()
+        super(CrawlSpider, self).__init__(*a, **kw)
 
         # auto follow start urls
         if self.start_urls:

scrapy/shell.py

@@ -55,7 +55,7 @@ class Shell(object):
             request = Request(url)
 
         spider = scrapymanager._create_spider_for_request(request, \
-            BaseSpider(), log_multiple=True)
+            BaseSpider('default'), log_multiple=True)
 
         print "Fetching %s..." % request
         response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \

scrapy/spider/models.py

@@ -52,7 +52,7 @@ class BaseSpider(object_ref):
         if not self.allowed_domains:
             self.allowed_domains = []
         if not self.name:
-            self.name = 'default'
+            raise ValueError("%s must have a name" % type(self).__name__)
 
         # XXX: SEP-12 forward compatibility (remove for 0.10)
         self.domain_name = self.name

scrapy/tests/test_contrib_exp_crawlspider_rules.py

@@ -109,7 +109,7 @@ class CompiledRuleInitializationTest(unittest.TestCase):
 
 class RulesTest(unittest.TestCase):
     def test_rules_manager_basic(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         response1 = HtmlResponse('http://example.org')
         response2 = HtmlResponse('http://othersite.org')
         rulesman = RulesManager([], spider)
@@ -134,7 +134,7 @@ class RulesTest(unittest.TestCase):
         self.failUnlessEqual(rule1.follow, True)
 
     def test_rules_manager_empty_rule(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         response = HtmlResponse('http://example.org')
 
         rulesman = RulesManager([Rule(follow=True)], spider)
@@ -144,7 +144,7 @@ class RulesTest(unittest.TestCase):
         self.failUnless(isinstance(rule.matcher, BaseMatcher))
 
     def test_rules_manager_default_matcher(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         response = HtmlResponse('http://example.org')
         callback = lambda x: None
 
@@ -156,7 +156,7 @@ class RulesTest(unittest.TestCase):
         self.failUnless(isinstance(rule.matcher, UrlMatcher))
 
     def test_rules_manager_matchers(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         response1 = HtmlResponse('http://example.org')
         response2 = HtmlResponse('http://othersite.org')
 
@@ -200,7 +200,7 @@ class RulesTest(unittest.TestCase):
     def test_rules_manager_callbacks(self):
         mycallback = lambda: True
 
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         spider.parse_item = lambda: True
 
         response1 = HtmlResponse('http://example.org')
@@ -229,7 +229,7 @@ class RulesTest(unittest.TestCase):
 
 
     def test_rules_manager_callback_with_arguments(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         response = HtmlResponse('http://example.org')
 
         kwargs = {'a': 1}

scrapy/tests/test_contrib_exp_crawlspider_spider.py

@@ -60,7 +60,7 @@ class CrawlSpiderTest(unittest.TestCase):
         Spider.request_extractors = extractors
         Spider.request_processors = processors
 
-        return Spider()
+        return Spider('foo')
 
     def test_start_url_auto_rule(self):
         spider = self.spider_factory()

scrapy/tests/test_downloader_handlers.py

@@ -32,11 +32,11 @@ class FileTestCase(unittest.TestCase):
 
         request = Request('file://%s' % self.tmpname + '^')
         assert request.url.upper().endswith('%5E')
-        return download_file(request, BaseSpider()).addCallback(_test)
+        return download_file(request, BaseSpider('foo')).addCallback(_test)
 
     def test_non_existent(self):
         request = Request('file://%s' % self.mktemp())
-        d = download_file(request, BaseSpider())
+        d = download_file(request, BaseSpider('foo'))
         return self.assertFailure(d, IOError)
 
 
@@ -66,20 +66,20 @@ class HttpTestCase(unittest.TestCase):
 
     def test_download(self):
         request = Request(self.getURL('file'))
-        d = download_http(request, BaseSpider())
+        d = download_http(request, BaseSpider('foo'))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEquals, "0123456789")
         return d
 
     def test_redirect_status(self):
         request = Request(self.getURL('redirect'))
-        d = download_http(request, BaseSpider())
+        d = download_http(request, BaseSpider('foo'))
         d.addCallback(lambda r: r.status)
         d.addCallback(self.assertEquals, 302)
         return d
 
     def test_timeout_download_from_spider(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         spider.download_timeout = 0.000001
         request = Request(self.getURL('wait'))
         d = download_http(request, spider)
@@ -91,7 +91,7 @@ class HttpTestCase(unittest.TestCase):
             self.assertEquals(request.headers, {})
 
         request = Request(self.getURL('host'))
-        return download_http(request, BaseSpider()).addCallback(_test)
+        return download_http(request, BaseSpider('foo')).addCallback(_test)
 
     def test_host_header_seted_in_request_headers(self):
         def _test(response):
@@ -99,9 +99,9 @@ class HttpTestCase(unittest.TestCase):
             self.assertEquals(request.headers.get('Host'), 'example.com')
 
         request = Request(self.getURL('host'), headers={'Host': 'example.com'})
-        return download_http(request, BaseSpider()).addCallback(_test)
+        return download_http(request, BaseSpider('foo')).addCallback(_test)
 
-        d = download_http(request, BaseSpider())
+        d = download_http(request, BaseSpider('foo'))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEquals, 'example.com')
         return d
@@ -109,14 +109,14 @@ class HttpTestCase(unittest.TestCase):
     def test_payload(self):
         body = '1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)
-        d = download_http(request, BaseSpider())
+        d = download_http(request, BaseSpider('foo'))
         d.addCallback(lambda r: r.body)
         d.addCallback(self.assertEquals, body)
         return d
 
     def test_broken_download(self):
         request = Request(self.getURL('broken'))
-        d = download_http(request, BaseSpider())
+        d = download_http(request, BaseSpider('foo'))
         return self.assertFailure(d, PartialDownloadError)
 
 
@@ -152,7 +152,7 @@ class HttpProxyTestCase(unittest.TestCase):
 
         http_proxy = self.getURL('')
         request = Request('https://example.com', meta={'proxy': http_proxy})
-        return download_http(request, BaseSpider()).addCallback(_test)
+        return download_http(request, BaseSpider('foo')).addCallback(_test)
 
     def test_download_without_proxy(self):
         def _test(response):
@@ -161,4 +161,4 @@ class HttpProxyTestCase(unittest.TestCase):
             self.assertEquals(response.body, '/path/to/resource')
 
         request = Request(self.getURL('path/to/resource'))
-        return download_http(request, BaseSpider()).addCallback(_test)
+        return download_http(request, BaseSpider('foo')).addCallback(_test)

scrapy/tests/test_downloadermiddleware_cookies.py

@@ -10,7 +10,7 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
 class CookiesMiddlewareTest(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = CookiesMiddleware()
 
     def tearDown(self):

scrapy/tests/test_downloadermiddleware_decompression.py

@@ -21,7 +21,7 @@ class DecompressionMiddlewareTest(TestCase):
 
     def setUp(self):
         self.mw = DecompressionMiddleware()
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
 
     def test_known_compression_formats(self):
         for fmt in self.test_formats:

scrapy/tests/test_downloadermiddleware_defaultheaders.py

@@ -9,7 +9,7 @@ from scrapy.spider import BaseSpider
 class TestDefaultHeadersMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = DefaultHeadersMiddleware()
         self.default_headers = dict([(k, [v]) for k, v in \
             settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])

scrapy/tests/test_downloadermiddleware_httpauth.py

@@ -18,7 +18,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
 
     def test_auth(self):
         self.mw.default_useragent = 'default_useragent'
-        spider = TestSpider()
+        spider = TestSpider('foo')
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, spider) is None
         self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==')

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -20,7 +20,7 @@ FORMAT = {
 class HttpCompressionTest(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = HttpCompressionMiddleware()
 
     def _getresponse(self, coding):

scrapy/tests/test_downloadermiddleware_httpproxy.py

@@ -8,7 +8,7 @@ from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
 from scrapy.conf import settings
 
-spider = BaseSpider()
+spider = BaseSpider('foo')
 
 class TestDefaultHeadersMiddleware(TestCase):
 

scrapy/tests/test_downloadermiddleware_redirect.py

@@ -8,7 +8,7 @@ from scrapy.http import Request, Response, HtmlResponse, Headers
 class RedirectMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = RedirectMiddleware()
 
     def test_priority_adjust(self):

scrapy/tests/test_downloadermiddleware_retry.py

@@ -10,7 +10,7 @@ from scrapy.http import Request, Response
 
 class RetryTest(unittest.TestCase):
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = RetryMiddleware()
         self.mw.max_retry_times = 2
 

scrapy/tests/test_downloadermiddleware_useragent.py

@@ -9,7 +9,7 @@ from scrapy.conf import settings
 class UserAgentMiddlewareTest(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = UserAgentMiddleware()
 
     def tearDown(self):

scrapy/tests/test_dupefilter.py

@@ -8,7 +8,7 @@ from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilt
 class RequestFingerprintDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         filter = RequestFingerprintDupeFilter()
         filter.open_spider(spider)
 
@@ -28,7 +28,7 @@ class RequestFingerprintDupeFilterTest(unittest.TestCase):
 class NullDupeFilterTest(unittest.TestCase):
 
     def test_filter(self):
-        spider = BaseSpider()
+        spider = BaseSpider('foo')
         filter = NullDupeFilter()
         filter.open_spider(spider)
 

scrapy/tests/test_schedulermiddleware_duplicatesfilter.py

@@ -10,7 +10,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
         self.mw = DuplicatesFilterMiddleware()
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw.open_spider(self.spider)
 
     def tearDown(self):

scrapy/tests/test_spider.py

@@ -9,7 +9,6 @@ from scrapy.spider import BaseSpider
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.contrib.spiders.crawl import CrawlSpider
 from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider
-from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter
 
 
 class BaseSpiderTest(unittest.TestCase):
@@ -77,10 +76,8 @@ class BaseSpiderTest(unittest.TestCase):
 
     def test_spider_without_name(self):
         """Constructor arguments are assigned to spider attributes"""
-        spider = self.spider_class()
-        self.assertEqual(spider.name, 'default')
-        spider = self.spider_class(foo='bar')
-        self.assertEqual(spider.foo, 'bar')
+        self.assertRaises(ValueError, self.spider_class)
+        self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
 
 class InitSpiderTest(BaseSpiderTest):

scrapy/tests/test_spidermiddleware_httperror.py

@@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware
 class TestHttpErrorMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = HttpErrorMiddleware()
         self.req = Request('http://scrapytest.org')
 

scrapy/tests/test_spidermiddleware_offsite.py

@@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware
 class TestOffsiteMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.spider.name = 'scrapytest.org'
         self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org']
 

scrapy/tests/test_spidermiddleware_referer.py

@@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.referer import RefererMiddleware
 class TestRefererMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = RefererMiddleware()
 
     def test_process_spider_output(self):

scrapy/tests/test_spidermiddleware_urlfilter.py

@@ -9,7 +9,7 @@ from scrapy.utils.url import canonicalize_url
 class TestUrlFilterMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = UrlFilterMiddleware()
 
     def test_process_spider_output(self):

scrapy/tests/test_spidermiddleware_urllength.py

@@ -12,7 +12,7 @@ class TestUrlLengthMiddleware(TestCase):
         settings.disabled = False
         settings.overrides['URLLENGTH_LIMIT'] = 25
 
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
         self.mw = UrlLengthMiddleware()
 
     def test_process_spider_output(self):

scrapy/tests/test_stats.py

@@ -9,7 +9,7 @@ from scrapy.stats.signals import stats_spider_opened, stats_spider_closing, \
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider()
+        self.spider = BaseSpider('foo')
 
     def test_collector(self):
         stats = StatsCollector()