diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 37fe0a873..b83f0fc48 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -118,21 +118,23 @@ class ExecutionEngine(object): if self.paused: return - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break - - if slot.start_requests and not self._needs_backout(spider): + while not self._needs_backout(spider) and slot.start_requests: try: request = next(slot.start_requests) + if request is None: + break + self.crawl(request, spider) + except StopIteration: slot.start_requests = None except Exception: slot.start_requests = None logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': spider}) - else: - self.crawl(request, spider) + + while not self._needs_backout(spider): + if not self._next_request_from_scheduler(spider): + break if self.spider_is_idle(spider) and slot.close_if_idle: self._spider_idle(spider)