Merge pull request #332 from nramirezuy/offsite-remove_multispider

offsite multispider support removed

scrapy/contrib/spidermiddleware/offsite.py

@@ -13,15 +13,10 @@ from scrapy import log
 
 class OffsiteMiddleware(object):
 
-    def __init__(self):
-        self.host_regexes = {}
-        self.domains_seen = {}
-
     @classmethod
     def from_crawler(cls, crawler):
         o = cls()
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
-        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
     def process_spider_output(self, response, result, spider):
@@ -31,15 +26,15 @@ class OffsiteMiddleware(object):
                     yield x
                 else:
                     domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen[spider]:
-                        self.domains_seen[spider].add(domain)
+                    if domain and domain not in self.domains_seen:
+                        self.domains_seen.add(domain)
                         log.msg(format="Filtered offsite request to %(domain)r: %(request)s",
                                 level=log.DEBUG, spider=spider, domain=domain, request=x)
             else:
                 yield x
 
     def should_follow(self, request, spider):
-        regex = self.host_regexes[spider]
+        regex = self.host_regex
         # hostanme can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ''
         return bool(regex.search(host))
@@ -54,9 +49,5 @@ class OffsiteMiddleware(object):
         return re.compile(regex)
 
     def spider_opened(self, spider):
-        self.host_regexes[spider] = self.get_host_regex(spider)
-        self.domains_seen[spider] = set()
-
-    def spider_closed(self, spider):
-        del self.host_regexes[spider]
-        del self.domains_seen[spider]
+        self.host_regex = self.get_host_regex(spider)
+        self.domains_seen = set()

scrapy/tests/test_spidermiddleware_offsite.py

@@ -29,9 +29,6 @@ class TestOffsiteMiddleware(TestCase):
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEquals(out, onsite_reqs)
 
-    def tearDown(self):
-        self.mw.spider_closed(self.spider)
-
 
 class TestOffsiteMiddleware2(TestOffsiteMiddleware):