From a44c990a59dd5238f5a3360532e69c049d49a91a Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 23 Apr 2018 10:46:19 +0000 Subject: [PATCH] consume start requests unit get a signal value --- scrapy/core/engine.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 37fe0a873..b83f0fc48 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -118,21 +118,23 @@ class ExecutionEngine(object): if self.paused: return - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break - - if slot.start_requests and not self._needs_backout(spider): + while not self._needs_backout(spider) and slot.start_requests: try: request = next(slot.start_requests) + if request is None: + break + self.crawl(request, spider) + except StopIteration: slot.start_requests = None except Exception: slot.start_requests = None logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': spider}) - else: - self.crawl(request, spider) + + while not self._needs_backout(spider): + if not self._next_request_from_scheduler(spider): + break if self.spider_is_idle(spider) and slot.close_if_idle: self._spider_idle(spider)