RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

looping over allowed_domains directly instead of via index

This commit is contained in:
Jana Cavojska 2017-11-20 21:23:31 +01:00
parent 62a6261028
commit 91ff194d1e
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/spidermiddlewares/offsite.py
View File

@ -52,10 +52,10 @@ class OffsiteMiddleware(object):
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
return re.compile('') # allow all by default
for domainIndex in range(0, len(allowed_domains)):
for domain in allowed_domains:
url_pattern = re.compile("^https?://.*$")
if url_pattern.match(allowed_domains[domainIndex]):
logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex])
if url_pattern.match(domain):
logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain)
regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
return re.compile(regex)