From 6ab3114d74978f814f4e4dd32fd693183b2f4b11 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 19 Feb 2020 12:43:53 +0000 Subject: [PATCH] move signal outside of spider --- scrapy/core/engine.py | 3 ++- scrapy/spiders/__init__.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 268ac75a5..98d14a50f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,6 +17,7 @@ from scrapy.http import Response, Request from scrapy.utils.misc import load_object from scrapy.utils.reactor import CallLaterOnce from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.spiders import WaitUntilQueueEmpty logger = logging.getLogger(__name__) @@ -121,7 +122,7 @@ class ExecutionEngine(object): while not self._needs_backout(spider) and slot.start_requests: try: request = next(slot.start_requests) - if request == spider.WaitUntilQueueEmpty: + if request == WaitUntilQueueEmpty: break except StopIteration: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 4ddd34760..51d9f2167 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -14,8 +14,10 @@ from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.python import iflatten +WaitUntilQueueEmpty = object() + + class Spider(object_ref): - WaitUntilQueueEmpty = object() """Base class for scrapy spiders. All spiders must inherit from this class. @@ -80,7 +82,7 @@ class Spider(object_ref): yield Request(url, dont_filter=True) def start_requests_with_control(self): - sig = self.WaitUntilQueueEmpty + sig = WaitUntilQueueEmpty gen_ = ((r, sig) if r != sig else (r,) for r in self.start_requests()) return iflatten(gen_)