From 830cd4f19f71a5171c1f0fd309e86e7d0b2bf157 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 24 Jun 2009 13:45:50 -0300 Subject: [PATCH] Restore download process queue processing after finish with recent transferred response --- scrapy/core/downloader/manager.py | 10 ++++++---- scrapy/core/engine.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 92d15d6cc..38d609d7f 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -109,7 +109,7 @@ class Downloader(object): # Process requests in queue if there are free slots to transfer for this site while site.queue and site.free_transfer_slots() > 0: request, deferred = site.queue.pop(0) - self._download(site, request, spider).chainDeferred(deferred) + self._download(site, request, spider, deferred) self._close_if_idle(domain) @@ -118,13 +118,15 @@ class Downloader(object): if site and site.closing and not site.active: del self.sites[domain] - def _download(self, site, request, spider): + def _download(self, site, request, spider, deferred): site.transferring.add(request) def _transferred(_): site.transferring.remove(request) self.process_queue(spider) - return _ - return mustbe_deferred(download_any, request, spider).addBoth(_transferred) + + dfd = mustbe_deferred(download_any, request, spider) + dfd.chainDeferred(deferred).addBoth(lambda _: deferred) + dfd.addBoth(_transferred) # request next download after finish processing current response def open_domain(self, domain): """Allocate resources to begin processing a domain""" diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7d9f0a34d..3ede2123d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -176,7 +176,9 @@ class ExecutionEngine(object): request, deferred = self.scheduler.next_request(domain) if request: spider = spiders.fromdomain(domain) - mustbe_deferred(self.download, request, spider).chainDeferred(deferred) + dwld = mustbe_deferred(self.download, request, spider) + dwld.chainDeferred(deferred).addBoth(lambda _: deferred) + dwld.addErrback(log.err) elif self.domain_is_idle(domain): self._domain_idle(domain)