From 6e9fa3a41ffbb97af226b72487ddf8b32af19a9d Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Fri, 24 Aug 2018 15:18:16 -0400 Subject: [PATCH] PEP8 ofsite middleware --- scrapy/spidermiddlewares/offsite.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 310166cad..232e96cbb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,7 +3,6 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ - import re import logging import warnings @@ -35,8 +34,9 @@ class OffsiteMiddleware(object): domain = urlparse_cached(x).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) - logger.debug("Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) self.stats.inc_value('offsite/domains', spider=spider) self.stats.inc_value('offsite/filtered', spider=spider) else: @@ -52,13 +52,15 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return re.compile('') # allow all by default + return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): - warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) - - regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) + message = ("allowed_domains accepts only domains, not URLs. " + "Ignoring URL entry %s in allowed_domains." % domain) + warnings.warn(message, URLWarning) + domains = [re.escape(d) for d in allowed_domains if d is not None] + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider):