RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix issue with responses preventing spiders to be idle in engine counts

This commit is contained in:
Daniel Grana 2011-07-15 13:57:24 -03:00
parent d207c0afe4
commit 4dadeb7ccb
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/core/engine.py
View File

@ -179,16 +179,14 @@ class ExecutionEngine(object):
def download(self, request, spider):
slot = self.slots[spider]
slot.add_request(request)
if isinstance(request, Response):
return request
d = self._download(request, spider)
d.addCallback(self.download, spider)
d.addBoth(self._remove_request, slot, request)
d.addBoth(self._downloaded, slot, request, spider)
return d
def _remove_request(self, _, slot, request):
def _downloaded(self, response, slot, request, spider):
slot.remove_request(request)
return _
return self.download(response, spider) \
if isinstance(response, Request) else response
def _download(self, request, spider):
slot = self.slots[spider]