diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 3b7f194e4..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -19,9 +19,6 @@ class OffsiteMiddleware(object): def __init__(self, stats): self.stats = stats - # default values - self.host_regex = re.compile('') # allow all by default - self.domains_seen = set() @classmethod def from_crawler(cls, crawler): @@ -55,7 +52,7 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return + return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): @@ -65,9 +62,8 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - host_regex = self.get_host_regex(spider) - if host_regex: - self.host_regex = host_regex + self.host_regex = self.get_host_regex(spider) + self.domains_seen = set() class URLWarning(Warning):