Don't filter out requests coming from spiders that don't define allowed_domains. Closes #225

docs/topics/spider-middleware.rst

@@ -224,6 +224,11 @@ OffsiteMiddleware
    printed. But if a request for ``someothersite.com`` is filtered, a message
    will be printed (but only for the first request filtred).
 
+   If the spider doesn't define an
+   :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute, or the
+   attribute is empty, the offsite middleware will allow all requests.
+
+
 RefererMiddleware
 -----------------
 

scrapy/contrib/spidermiddleware/offsite.py

@@ -40,14 +40,17 @@ class OffsiteMiddleware(object):
         host = urlparse_cached(request).hostname or ''
         return bool(regex.search(host))
 
-    def get_host_regex(self, domains):
+    def get_host_regex(self, spider):
         """Override this method to implement a different offsite policy"""
-        domains = [d.replace('.', r'\.') for d in domains]
+        allowed_domains = getattr(spider, 'allowed_domains', None)
+        if not allowed_domains:
+            return re.compile('') # allow all by default
+        domains = [d.replace('.', r'\.') for d in allowed_domains]
         regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
         return re.compile(regex)
 
     def spider_opened(self, spider):
-        self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains)
+        self.host_regexes[spider] = self.get_host_regex(spider)
         self.domains_seen[spider] = set()
 
     def spider_closed(self, spider):

scrapy/tests/test_spidermiddleware_offsite.py

@@ -8,13 +8,13 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware
 class TestOffsiteMiddleware(TestCase):
 
     def setUp(self):
-        self.spider = BaseSpider('foo')
-        self.spider.name = 'scrapytest.org'
-        self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org']
-
+        self.spider = self._get_spider()
         self.mw = OffsiteMiddleware()
         self.mw.spider_opened(self.spider)
 
+    def _get_spider(self):
+        return BaseSpider('foo', allowed_domains=['scrapytest.org', 'scrapy.org'])
+
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
 
@@ -30,3 +30,20 @@ class TestOffsiteMiddleware(TestCase):
     def tearDown(self):
         self.mw.spider_closed(self.spider)
 
+
+class TestOffsiteMiddleware2(TestOffsiteMiddleware):
+
+    def _get_spider(self):
+        return BaseSpider('foo', allowed_domains=None)
+
+    def test_process_spider_output(self):
+        res = Response('http://scrapytest.org')
+        reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')]
+        out = list(self.mw.process_spider_output(res, reqs, self.spider))
+        self.assertEquals(out, reqs)
+
+class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
+
+    def _get_spider(self):
+        return BaseSpider('foo')
+