From 91ff194d1e9477d2196817ea1dc8beb220c3e058 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Mon, 20 Nov 2017 21:23:31 +0100 Subject: [PATCH] looping over allowed_domains directly instead of via index --- scrapy/spidermiddlewares/offsite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f51b0a2b0..8ff35e29f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,10 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - for domainIndex in range(0, len(allowed_domains)): + for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") - if url_pattern.match(allowed_domains[domainIndex]): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) + if url_pattern.match(domain): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex)