core: strip not working close delay support from core to (working) extension

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401069

scrapy/trunk/scrapy/contrib/delayedclosedomain.py

@@ -0,0 +1,37 @@
+"""
+DelayedCloseDomain is an extension that keeps open a domain until a
+configurable amount of idle time is reached
+"""
+
+from datetime import datetime
+
+from pydispatch import dispatcher
+from collections import defaultdict
+
+from scrapy.core import signals
+from scrapy.core.engine import scrapyengine
+from scrapy.core.exceptions import NotConfigured, DontCloseDomain
+from scrapy.conf import settings
+
+
+class DelayedCloseDomain(object):
+    def __init__(self):
+        self.delay = settings.getint('DOMAIN_CLOSE_DELAY')
+        if not self.delay:
+            raise NotConfigured
+
+        self.opened_at = defaultdict(datetime.now)
+        dispatcher.connect(self.domain_idle, signal=signals.domain_idle)
+        dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
+
+    def domain_idle(self, domain):
+        lastseen = scrapyengine.downloader.lastseen(domain)
+        if not lastseen:
+            lastseen = self.opened_at[domain]
+
+        delta = datetime.now() - lastseen
+        if delta.seconds < self.delay:
+            raise DontCloseDomain
+
+    def domain_closed(self, domain):
+        self.opened_at.pop(domain, None)

scrapy/trunk/scrapy/core/engine.py

@@ -49,7 +49,6 @@ class ExecutionEngine(object):
     def __init__(self):
         self.configured = False
         self.keep_alive = False
-        self.domain_close_delay = settings.getint('DOMAIN_CLOSE_DELAY') # seconds after an idle domain will be closed
         self.initializing = set() # domais in intialization state
         self.cancelled = set() # domains in cancelation state
         self.debug_mode = settings.getbool('ENGINE_DEBUG')
@@ -211,17 +210,7 @@ class ExecutionEngine(object):
         downloading = not self.downloader.domain_is_idle(domain)
         haspipe = not self.pipeline.domain_is_idle(domain)
         oninit = domain in self.initializing
-        if pending or downloading or haspipe or oninit or scraping:
-            return False
-        elif self.domain_close_delay:
-            lastseen = self.downloader.lastseen(domain)
-            if not lastseen: # domain not yet started
-                return False
-            now = datetime.now()
-            delta = now - lastseen
-            return (delta.seconds >= self.domain_close_delay)
-        else:
-            return True
+        return not (pending or downloading or haspipe or oninit or scraping)
 
     def domain_is_open(self, domain):
         return domain in self.downloader.sites
@@ -319,12 +308,6 @@ class ExecutionEngine(object):
             if not self.next_domain():
                 return self._stop_if_idle()
 
-        # purge idle domains - (domain_close_delay support)
-        if self.domain_close_delay:
-            for domain in self.downloader.sites:
-                if self.domain_is_idle(domain):
-                    self._domain_idle(domain)
-
     def _add_starter(self, request, spider, domain_priority):
         domain = spider.domain_name
         if not self.scheduler.is_pending(domain):