From 22c68baf990f15d249f38c481f24a984977be3e5 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Thu, 7 Dec 2017 18:38:29 +0100 Subject: [PATCH] url_pattern is now being compiled before entering the loop --- scrapy/spidermiddlewares/offsite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f595eef42..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -53,8 +53,8 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: - url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) @@ -67,4 +67,4 @@ class OffsiteMiddleware(object): class URLWarning(Warning): - pass \ No newline at end of file + pass