RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

consume start requests unit get a signal value

This commit is contained in:
Vostretsov Nikita 2018-04-23 10:46:19 +00:00
parent 49e6d67ba1
commit a44c990a59
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/core/engine.py
View File

@ -118,21 +118,23 @@ class ExecutionEngine(object):
if self.paused:
return
while not self._needs_backout(spider):
if not self._next_request_from_scheduler(spider):
break
if slot.start_requests and not self._needs_backout(spider):
while not self._needs_backout(spider) and slot.start_requests:
try:
request = next(slot.start_requests)
if request is None:
break
self.crawl(request, spider)
except StopIteration:
slot.start_requests = None
except Exception:
slot.start_requests = None
logger.error('Error while obtaining start requests',
exc_info=True, extra={'spider': spider})
else:
self.crawl(request, spider)
while not self._needs_backout(spider):
if not self._next_request_from_scheduler(spider):
break
if self.spider_is_idle(spider) and slot.close_if_idle:
self._spider_idle(spider)