diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 3ce50ae28..f8acbf8d8 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -44,8 +44,7 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - domains = [d.replace('.', r'\.') for d in allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains) return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index b7f09dd02..fd6e43753 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -23,7 +23,11 @@ class TestOffsiteMiddleware(TestCase): Request('http://sub.scrapy.org/1'), Request('http://offsite.tld/letmepass', dont_filter=True)] offsite_reqs = [Request('http://scrapy2.org'), - Request('http://offsite.tld/')] + Request('http://offsite.tld/'), + Request('http://offsite.tld/scrapytest.org'), + Request('http://offsite.tld/rogue.scrapytest.org'), + Request('http://rogue.scrapytest.org.haha.com'), + Request('http://roguescrapytest.org')] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider))