RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Restore download process queue processing after finish with recent transferred response

This commit is contained in:
Daniel Grana 2009-06-24 13:45:50 -03:00
parent 87df33ce0a
commit 830cd4f19f
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/core/downloader/manager.py
View File

@ -109,7 +109,7 @@ class Downloader(object):
# Process requests in queue if there are free slots to transfer for this site
while site.queue and site.free_transfer_slots() > 0:
request, deferred = site.queue.pop(0)
self._download(site, request, spider).chainDeferred(deferred)
self._download(site, request, spider, deferred)
self._close_if_idle(domain)
@ -118,13 +118,15 @@ class Downloader(object):
if site and site.closing and not site.active:
del self.sites[domain]
def _download(self, site, request, spider):
def _download(self, site, request, spider, deferred):
site.transferring.add(request)
def _transferred(_):
site.transferring.remove(request)
self.process_queue(spider)
return _
return mustbe_deferred(download_any, request, spider).addBoth(_transferred)
dfd = mustbe_deferred(download_any, request, spider)
dfd.chainDeferred(deferred).addBoth(lambda _: deferred)
dfd.addBoth(_transferred) # request next download after finish processing current response
def open_domain(self, domain):
"""Allocate resources to begin processing a domain"""

4
scrapy/core/engine.py
View File

@ -176,7 +176,9 @@ class ExecutionEngine(object):
request, deferred = self.scheduler.next_request(domain)
if request:
spider = spiders.fromdomain(domain)
mustbe_deferred(self.download, request, spider).chainDeferred(deferred)
dwld = mustbe_deferred(self.download, request, spider)
dwld.chainDeferred(deferred).addBoth(lambda _: deferred)
dwld.addErrback(log.err)
elif self.domain_is_idle(domain):
self._domain_idle(domain)