diff --git a/scrapy/trunk/scrapy/contrib/delayedclosedomain.py b/scrapy/trunk/scrapy/contrib/delayedclosedomain.py new file mode 100644 index 000000000..ef79deaaa --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/delayedclosedomain.py @@ -0,0 +1,37 @@ +""" +DelayedCloseDomain is an extension that keeps open a domain until a +configurable amount of idle time is reached +""" + +from datetime import datetime + +from pydispatch import dispatcher +from collections import defaultdict + +from scrapy.core import signals +from scrapy.core.engine import scrapyengine +from scrapy.core.exceptions import NotConfigured, DontCloseDomain +from scrapy.conf import settings + + +class DelayedCloseDomain(object): + def __init__(self): + self.delay = settings.getint('DOMAIN_CLOSE_DELAY') + if not self.delay: + raise NotConfigured + + self.opened_at = defaultdict(datetime.now) + dispatcher.connect(self.domain_idle, signal=signals.domain_idle) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + def domain_idle(self, domain): + lastseen = scrapyengine.downloader.lastseen(domain) + if not lastseen: + lastseen = self.opened_at[domain] + + delta = datetime.now() - lastseen + if delta.seconds < self.delay: + raise DontCloseDomain + + def domain_closed(self, domain): + self.opened_at.pop(domain, None) diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index 349ec496c..6b0ac19fc 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -49,7 +49,6 @@ class ExecutionEngine(object): def __init__(self): self.configured = False self.keep_alive = False - self.domain_close_delay = settings.getint('DOMAIN_CLOSE_DELAY') # seconds after an idle domain will be closed self.initializing = set() # domais in intialization state self.cancelled = set() # domains in cancelation state self.debug_mode = settings.getbool('ENGINE_DEBUG') @@ -211,17 +210,7 @@ class ExecutionEngine(object): downloading = not self.downloader.domain_is_idle(domain) haspipe = not self.pipeline.domain_is_idle(domain) oninit = domain in self.initializing - if pending or downloading or haspipe or oninit or scraping: - return False - elif self.domain_close_delay: - lastseen = self.downloader.lastseen(domain) - if not lastseen: # domain not yet started - return False - now = datetime.now() - delta = now - lastseen - return (delta.seconds >= self.domain_close_delay) - else: - return True + return not (pending or downloading or haspipe or oninit or scraping) def domain_is_open(self, domain): return domain in self.downloader.sites @@ -319,12 +308,6 @@ class ExecutionEngine(object): if not self.next_domain(): return self._stop_if_idle() - # purge idle domains - (domain_close_delay support) - if self.domain_close_delay: - for domain in self.downloader.sites: - if self.domain_is_idle(domain): - self._domain_idle(domain) - def _add_starter(self, request, spider, domain_priority): domain = spider.domain_name if not self.scheduler.is_pending(domain):