RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

speed up consumption of spider start requests, while the engine has capacity to process them

This commit is contained in:
Pablo Hoffman 2011-07-20 01:18:00 -03:00
parent 9f742fc97c
commit de0bf22010
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/core/engine.py
View File

@ -108,14 +108,16 @@ class ExecutionEngine(object):
if not self._next_request(spider):
break
if self.spider_is_idle(spider):
slot = self.slots[spider]
slot = self.slots[spider]
if slot.requests and not self._needs_backout(spider):
try:
request = slot.requests.next()
self.crawl(request, spider)
except StopIteration:
if slot.close_if_idle:
self._spider_idle(spider)
slot.requests = None
if self.spider_is_idle(spider) and slot.close_if_idle:
self._spider_idle(spider)
def _needs_backout(self, spider):
slot = self.slots[spider]