From 62a626102877de4998538717f34e61d2f7d2622c Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sat, 18 Nov 2017 20:03:59 +0100 Subject: [PATCH] Issues a warning when user puts a URL into allowed_domains (#2250) --- scrapy/spidermiddlewares/offsite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index ea1c9270f..f51b0a2b0 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,6 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + for domainIndex in range(0, len(allowed_domains)): + url_pattern = re.compile("^https?://.*$") + if url_pattern.match(allowed_domains[domainIndex]): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex)