diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index a6f8feb2e..dfa1aca77 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -259,7 +259,7 @@ class ExecutionEngine: return while not self.needs_backout(): - if self._start_scheduled_request() is None: + if not self._start_scheduled_request(): break if self.spider_is_idle() and self._slot.close_if_idle: @@ -294,7 +294,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _start_scheduled_request(self) -> Deferred[None] | None: + def _start_scheduled_request(self) -> bool: assert self._slot is not None # typing assert self.spider is not None # typing assert self.scheduler is not None # typing @@ -309,7 +309,7 @@ class ExecutionEngine: return None if request is None: self.signals.send_catch_log(signals.scheduler_empty) - return None + return False d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) @@ -342,7 +342,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - return d2 + return True def _handle_downloader_output( self, result: Request | Response | Failure, request: Request diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index d9e176ed8..394fd76da 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -3,7 +3,6 @@ from asyncio import sleep import pytest from testfixtures import LogCapture -from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from scrapy import Spider, signals @@ -12,18 +11,12 @@ from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler from .test_scheduler import MemoryScheduler +from .utils import twisted_sleep -SLEEP_SECONDS = 0.1 ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} - -def twisted_sleep(seconds): - from twisted.internet import reactor - - d = Deferred() - reactor.callLater(seconds, d.callback, None) - return d +SLEEP_SECONDS = 0.1 class MainTestCase(TestCase): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 95bcf48d8..fb749d56d 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -11,8 +11,9 @@ from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler +from tests.test_spider_start import SLEEP_SECONDS -from .test_spider_start import SLEEP_SECONDS, twisted_sleep +from .utils import twisted_sleep ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29bb..e5e56f414 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,9 @@ +from twisted.internet.defer import Deferred + + +def twisted_sleep(seconds): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(seconds, d.callback, None) + return d