diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index a32672a99..511becc60 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -13,15 +13,10 @@ from scrapy import log class OffsiteMiddleware(object): - def __init__(self): - self.host_regexes = {} - self.domains_seen = {} - @classmethod def from_crawler(cls, crawler): o = cls() crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) - crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o def process_spider_output(self, response, result, spider): @@ -31,15 +26,15 @@ class OffsiteMiddleware(object): yield x else: domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen[spider]: - self.domains_seen[spider].add(domain) + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) log.msg(format="Filtered offsite request to %(domain)r: %(request)s", level=log.DEBUG, spider=spider, domain=domain, request=x) else: yield x def should_follow(self, request, spider): - regex = self.host_regexes[spider] + regex = self.host_regex # hostanme can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or '' return bool(regex.search(host)) @@ -54,9 +49,5 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - self.host_regexes[spider] = self.get_host_regex(spider) - self.domains_seen[spider] = set() - - def spider_closed(self, spider): - del self.host_regexes[spider] - del self.domains_seen[spider] + self.host_regex = self.get_host_regex(spider) + self.domains_seen = set() diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 44433ee57..f7523f726 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -29,9 +29,6 @@ class TestOffsiteMiddleware(TestCase): out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEquals(out, onsite_reqs) - def tearDown(self): - self.mw.spider_closed(self.spider) - class TestOffsiteMiddleware2(TestOffsiteMiddleware):