diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6731c59b7..6d68d0a27 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -121,7 +121,7 @@ class ExecutionEngine(object): while not self._needs_backout(spider) and slot.start_requests: try: request = next(slot.start_requests) - if request == signals.WaitUntilQueueEmpty: + if request == spider.WaitUntilQueueEmpty: break self.crawl(request, spider) diff --git a/scrapy/signals.py b/scrapy/signals.py index fe724ce80..de0886fb6 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -26,5 +26,3 @@ stats_spider_closed = spider_closed item_passed = item_scraped request_received = request_scheduled - -WaitUntilQueueEmpty = object() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 4e214ec16..80a36cf6d 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -17,6 +17,8 @@ from scrapy.utils.python import iflatten class Spider(object_ref): + WaitUntilQueueEmpty = object() + """Base class for scrapy spiders. All spiders must inherit from this class. """ @@ -84,7 +86,7 @@ class Spider(object_ref): yield Request(url, dont_filter=True) def start_requests_with_control(self): - sig = signals.WaitUntilQueueEmpty + sig = self.WaitUntilQueueEmpty gen_ = ((r, sig) if r != sig else (r,) for r in self.start_requests()) return iflatten(gen_)