diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b83f0fc48..6731c59b7 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -121,7 +121,7 @@ class ExecutionEngine(object): while not self._needs_backout(spider) and slot.start_requests: try: request = next(slot.start_requests) - if request is None: + if request == signals.WaitUntilQueueEmpty: break self.crawl(request, spider) diff --git a/scrapy/signals.py b/scrapy/signals.py index de0886fb6..fe724ce80 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -26,3 +26,5 @@ stats_spider_closed = spider_closed item_passed = item_scraped request_received = request_scheduled + +WaitUntilQueueEmpty = object() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index a522c6af9..0c7eefc3e 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -86,8 +86,8 @@ class Spider(object_ref): for r in self.start_requests(): yield r - if r is not None: - yield None + if r != signals.WaitUntilQueueEmpty: + yield signals.WaitUntilQueueEmpty def make_requests_from_url(self, url): """ This method is deprecated. """