From eefa878f98e404fd00af017e58c97261968f9067 Mon Sep 17 00:00:00 2001 From: Cameron Lane Date: Thu, 12 Jun 2014 11:42:14 -0400 Subject: [PATCH 1/2] [#744] Ensure domain is not None before building regex --- scrapy/contrib/spidermiddleware/offsite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 95702d777..136714508 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -49,7 +49,7 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) def spider_opened(self, spider): From a069d366529d8fa9d4dcdfcc8bfc88a90d83e4b0 Mon Sep 17 00:00:00 2001 From: Cameron Lane Date: Thu, 12 Jun 2014 11:43:03 -0400 Subject: [PATCH 2/2] [#744] Test for allowed domains including NoneTypes --- scrapy/tests/test_spidermiddleware_offsite.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 8477e4dee..7e0d248dc 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -5,6 +5,7 @@ from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler +from urlparse import urlparse class TestOffsiteMiddleware(TestCase): @@ -52,3 +53,15 @@ class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spider(self): return Spider('foo') + +class TestOffsiteMiddleware4(TestOffsiteMiddleware3): + + def _get_spider(self): + bad_hostname = urlparse('http:////scrapytest.org').hostname + return Spider('foo', allowed_domains=['scrapytest.org', None, bad_hostname]) + + def test_process_spider_output(self): + res = Response('http://scrapytest.org') + reqs = [Request('http://scrapytest.org/1')] + out = list(self.mw.process_spider_output(res, reqs, self.spider)) + self.assertEquals(out, reqs)