move signal outside of spider

scrapy/core/engine.py

@@ -17,6 +17,7 @@ from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
+from scrapy.spiders import WaitUntilQueueEmpty
 
 logger = logging.getLogger(__name__)
 
@@ -121,7 +122,7 @@ class ExecutionEngine(object):
         while not self._needs_backout(spider) and slot.start_requests:
             try:
                 request = next(slot.start_requests)
-                if request == spider.WaitUntilQueueEmpty:
+                if request == WaitUntilQueueEmpty:
                     break
 
             except StopIteration:

scrapy/spiders/__init__.py

@@ -14,8 +14,10 @@ from scrapy.utils.deprecate import method_is_overridden
 from scrapy.utils.python import iflatten
 
 
+WaitUntilQueueEmpty = object()
+
+
 class Spider(object_ref):
-    WaitUntilQueueEmpty = object()
 
     """Base class for scrapy spiders. All spiders must inherit from this
     class.
@@ -80,7 +82,7 @@ class Spider(object_ref):
                 yield Request(url, dont_filter=True)
 
     def start_requests_with_control(self):
-        sig = self.WaitUntilQueueEmpty
+        sig = WaitUntilQueueEmpty
         gen_ = ((r, sig) if r != sig else (r,) for r in self.start_requests())
         return iflatten(gen_)