diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index da3c9d5f9..9ba853814 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -224,6 +224,11 @@ OffsiteMiddleware printed. But if a request for ``someothersite.com`` is filtered, a message will be printed (but only for the first request filtred). + If the spider doesn't define an + :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + RefererMiddleware ----------------- diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 32190f6a1..ac583c5a5 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -40,14 +40,17 @@ class OffsiteMiddleware(object): host = urlparse_cached(request).hostname or '' return bool(regex.search(host)) - def get_host_regex(self, domains): + def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" - domains = [d.replace('.', r'\.') for d in domains] + allowed_domains = getattr(spider, 'allowed_domains', None) + if not allowed_domains: + return re.compile('') # allow all by default + domains = [d.replace('.', r'\.') for d in allowed_domains] regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): - self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains) + self.host_regexes[spider] = self.get_host_regex(spider) self.domains_seen[spider] = set() def spider_closed(self, spider): diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 861595524..8d9c173b4 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -8,13 +8,13 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider('foo') - self.spider.name = 'scrapytest.org' - self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] - + self.spider = self._get_spider() self.mw = OffsiteMiddleware() self.mw.spider_opened(self.spider) + def _get_spider(self): + return BaseSpider('foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -30,3 +30,20 @@ class TestOffsiteMiddleware(TestCase): def tearDown(self): self.mw.spider_closed(self.spider) + +class TestOffsiteMiddleware2(TestOffsiteMiddleware): + + def _get_spider(self): + return BaseSpider('foo', allowed_domains=None) + + def test_process_spider_output(self): + res = Response('http://scrapytest.org') + reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')] + out = list(self.mw.process_spider_output(res, reqs, self.spider)) + self.assertEquals(out, reqs) + +class TestOffsiteMiddleware3(TestOffsiteMiddleware2): + + def _get_spider(self): + return BaseSpider('foo') +