special object for signaling

scrapy/core/engine.py

@@ -121,7 +121,7 @@ class ExecutionEngine(object):
         while not self._needs_backout(spider) and slot.start_requests:
             try:
                 request = next(slot.start_requests)
-                if request is None:
+                if request == signals.WaitUntilQueueEmpty:
                     break
                 self.crawl(request, spider)
 

scrapy/signals.py

@@ -26,3 +26,5 @@ stats_spider_closed = spider_closed
 item_passed = item_scraped
 
 request_received = request_scheduled
+
+WaitUntilQueueEmpty = object()

scrapy/spiders/__init__.py

@@ -86,8 +86,8 @@ class Spider(object_ref):
         for r in self.start_requests():
             yield r
 
-            if r is not None:
-                yield None
+            if r != signals.WaitUntilQueueEmpty:
+                yield signals.WaitUntilQueueEmpty
 
     def make_requests_from_url(self, url):
         """ This method is deprecated. """