diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 04f2be5fa..92d15d6cc 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -50,13 +50,7 @@ class Downloader(object): parallel. """ - def __init__(self, engine): - """Create the downlaoder. - - ``engine`` is the scrapy engine controlling this downloader - """ - - self.engine = engine + def __init__(self): self.sites = {} self.middleware = DownloaderMiddlewareManager() self.concurrent_domains = settings.getint('CONCURRENT_DOMAINS') @@ -123,7 +117,6 @@ class Downloader(object): site = self.sites.get(domain) if site and site.closing and not site.active: del self.sites[domain] - self.engine.closed_domain(domain) # notify engine. def _download(self, site, request, spider): site.transferring.add(request) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 20e2e53a7..443f8f865 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -62,7 +62,7 @@ class ExecutionEngine(object): """ self.scheduler = scheduler or Scheduler() self.domain_scheduler = load_object(settings['DOMAIN_SCHEDULER'])() - self.downloader = downloader or Downloader(self) + self.downloader = downloader or Downloader() self.spidermiddleware = SpiderMiddlewareManager() self._scraping = {} self.pipeline = ItemPipelineManager() @@ -361,7 +361,6 @@ class ExecutionEngine(object): return except: log.exc("Exception catched on domain_idle signal dispatch") - if self.domain_is_idle(domain): self.close_domain(domain, reason='finished') @@ -376,19 +375,23 @@ class ExecutionEngine(object): log.msg("Closing domain (%s)" % reason, domain=domain) self.closing[domain] = reason self.downloader.close_domain(domain) + self.scheduler.clear_pending_requests(domain) + self._finish_closing_domain_if_idle(domain) - def closed_domain(self, domain): - """ - This function is called after the domain has been closed, and throws - the domain_closed signal which is meant to be used for cleaning up - purposes. In contrast to domain_idle, this function is called only - ONCE for each domain run. - """ + def _finish_closing_domain_if_idle(self, domain): + """Call _finish_closing_domain if domain is idle""" + if self.domain_is_idle(domain): + self._finish_closing_domain(domain) + else: + reactor.callLater(5, self._finish_closing_domain_if_idle, domain) + + def _finish_closing_domain(self, domain): + """This function is called after the domain has been closed""" spider = spiders.fromdomain(domain) self.scheduler.close_domain(domain) self.pipeline.close_domain(domain) del self._scraping[domain] - reason = self.closing[domain] + reason = self.closing.get(domain, 'finished') signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, reason=reason) log.msg("Domain closed (%s)" % reason, domain=domain) self.closing.pop(domain, None)