RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #478 from redapple/offsitetests 

Add tests for OffsiteMiddleware() + use re.escape() in domains regexp
This commit is contained in:
Daniel Graña 2014-01-16 14:02:26 -08:00
parent 151f9478c1 13002564b9
commit f0851e41ec
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/contrib/spidermiddleware/offsite.py
View File

@ -44,8 +44,7 @@ class OffsiteMiddleware(object):
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
return re.compile('') # allow all by default
domains = [d.replace('.', r'\.') for d in allowed_domains]
regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains)
return re.compile(regex)
def spider_opened(self, spider):

6
scrapy/tests/test_spidermiddleware_offsite.py
View File

@ -23,7 +23,11 @@ class TestOffsiteMiddleware(TestCase):
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True)]
offsite_reqs = [Request('http://scrapy2.org'),
Request('http://offsite.tld/')]
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org')]
reqs = onsite_reqs + offsite_reqs
out = list(self.mw.process_spider_output(res, reqs, self.spider))